197 lines
6.1 KiB
Python
197 lines
6.1 KiB
Python
from flask import Flask, render_template, request
|
|
from datetime import datetime
|
|
from probeutils import utils as probeutils
|
|
import re
|
|
import fnmatch
|
|
import os
|
|
import meinheld
|
|
|
|
|
|
app = Flask(__name__)
|
|
|
|
app.config['UPLOAD_FOLDER'] = "./dataset/"
|
|
|
|
app.config['MAX_CONTENT_LENGTH'] = 10000000000 # 10GB
|
|
meinheld.set_max_content_length(100*1024*1024)
|
|
|
|
app.config['DOWNLOADS_URL'] = "https://download.med.upct.es/"
|
|
|
|
probes = [{
|
|
'sensor': 'SUNA',
|
|
'img': 'suna.jpg',
|
|
'active': ['SATSLF0037'],
|
|
'stations': ['M1', 'M2', 'M3']
|
|
}, {
|
|
'sensor': 'FIRe',
|
|
'img': 'fire.jpg',
|
|
'active': ['SATFIS0006'],
|
|
'stations': ['M1', 'M2', 'M3']
|
|
}, {
|
|
'sensor': 'PhycoCTD',
|
|
'img': 'phycoctd.jpg',
|
|
'active': ['phyco_v1', 'phyco_v2'],
|
|
'stations': ['M1', 'M2', 'M3']
|
|
}, {
|
|
'sensor': 'CastAway',
|
|
'img': 'castaway.jpg',
|
|
'active': ['CC1326008'],
|
|
'stations': ['M1', 'M2', 'M3']
|
|
}
|
|
#,{
|
|
# 'sensor': 'GoPro',
|
|
# 'img': 'gopro.png',
|
|
# 'active': ['gopro1'],
|
|
# 'stations': ['M1', 'M2', 'M3']
|
|
#}
|
|
]
|
|
|
|
# Return our beautiful Bootstrap webpage. That we totally have.
|
|
@app.route('/')
|
|
def upload():
|
|
return render_template('upload.html', probes=probes)
|
|
|
|
# What happens when the files just don't fit
|
|
@app.errorhandler(413)
|
|
def request_entity_too_large(error):
|
|
return 'File Too Thicc', 413
|
|
|
|
# What happens when we did an oopsie
|
|
@app.errorhandler(500)
|
|
def internal_server_error(error):
|
|
return render_template('error.html')
|
|
|
|
# What happens when someone uploads a thingy
|
|
@app.route('/uploader', methods=['POST'])
|
|
def upload_file():
|
|
if request.method == 'POST':
|
|
f = request.files['file']
|
|
|
|
# Get which probe the user uploaded
|
|
probe = request.form.get('probe')
|
|
|
|
# Get selected probe
|
|
activeProbe = request.form.get('activeProbe')
|
|
|
|
# Check if its a valid file to upload
|
|
# TODO: Implement this method
|
|
# probeutils.check(probe, f)
|
|
|
|
# Check if folder exists
|
|
try:
|
|
if not os.path.exists(
|
|
os.path.join(app.config['UPLOAD_FOLDER'], probe)):
|
|
os.makedirs(os.path.join(app.config['UPLOAD_FOLDER'], probe))
|
|
|
|
if not os.path.exists(
|
|
os.path.join(app.config['UPLOAD_FOLDER'], probe, 'raw')):
|
|
os.makedirs(os.path.join(
|
|
app.config['UPLOAD_FOLDER'], probe, 'raw'))
|
|
except Exception:
|
|
return render_template('error.html')
|
|
|
|
# Strip station
|
|
# if forceStation is checked, override all station parsing
|
|
if (request.form.get("forceStation" + probe) != None):
|
|
station = request.form.get("stations" + probe)
|
|
else:
|
|
if (fnmatch.fnmatch((f.filename).upper(), '*M1*') or fnmatch.fnmatch((f.filename).upper(), 'M1*')):
|
|
station = 'M1'
|
|
elif (fnmatch.fnmatch((f.filename).upper(), "*M2*") or fnmatch.fnmatch((f.filename).upper(), "M2*")):
|
|
station = 'M2'
|
|
elif (fnmatch.fnmatch((f.filename).upper(), "*M3*") or fnmatch.fnmatch((f.filename).upper(), "M3*")):
|
|
station = 'M3'
|
|
|
|
# Date Parser of filename
|
|
match1 = re.search('\d{4}-\d{2}-\d{2}', f.filename)
|
|
if (match1 == None):
|
|
match1 = re.search('\d{2}-\d{2}-\d{4}', f.filename)
|
|
try:
|
|
date = datetime.strptime(match1.group(), '%d-%m-%Y').date()
|
|
except Exception:
|
|
# Error! More strange data format XD
|
|
match1 = re.search('\d{4}\d{2}\d{2}', f.filename)
|
|
if (match1 == None):
|
|
match1 = re.search('\d{2}\d{2}\d{4}', f.filename)
|
|
date = datetime.strptime(match1.group(), '%d%m%Y').date()
|
|
else:
|
|
date = datetime.strptime(match1.group(), '%Y%m%d').date()
|
|
else:
|
|
date = datetime.strptime(match1.group(), '%Y-%m-%d').date()
|
|
|
|
# Crafting the real overpowered name!!!!
|
|
filename = probe + '-' + station + '-'+ activeProbe + '-'+ date.strftime("%Y-%m-%d") + '.csv'
|
|
#print(filename)
|
|
|
|
# Setting the filePath and saving it
|
|
finalFilename = 'raw-'+filename
|
|
rawFilePath = os.path.join(app.config['UPLOAD_FOLDER'], probe, 'raw', finalFilename)
|
|
f.save(rawFilePath)
|
|
|
|
# Start with probe definition
|
|
if (probe == 'CastAway'):
|
|
''' CastAway Files '''
|
|
|
|
df_table = probeutils.process_castaway(rawFilePath)
|
|
|
|
elif (probe == 'SUNA'):
|
|
''' SUNA Files '''
|
|
|
|
df_table = probeutils.process_suna(rawFilePath)
|
|
|
|
elif (probe == 'FIRe'):
|
|
''' FIRe Files '''
|
|
|
|
df_table = probeutils.process_fire(rawFilePath)
|
|
|
|
elif (probe == 'PhycoCTD'):
|
|
''' PhycoCTD Files '''
|
|
|
|
df_table = probeutils.process_phyco(rawFilePath)
|
|
else:
|
|
''' Empty probe '''
|
|
|
|
df_table = "<div></div>"
|
|
|
|
# Save file on raw folder and create URL download
|
|
url_download = app.config['DOWNLOADS_URL'] + probe + "/" + finalFilename
|
|
|
|
# Return success webpage
|
|
return render_template('successful.html', url=url_download, tables=df_table)
|
|
else:
|
|
# No POST Found
|
|
return render_template('error.html')
|
|
|
|
## Force regenerate QC
|
|
@app.route('/regenerate')
|
|
def regenerate():
|
|
error_list = []
|
|
#for each file in raw folder, execute all QC
|
|
PATH = './dataset/FIRe/raw'
|
|
for filename in os.listdir(PATH):
|
|
try:
|
|
filename_nn = probeutils.execution_NN(filename)
|
|
probeutils.execution_PAR(filename_nn)
|
|
except Exception as err:
|
|
#print(filename)
|
|
#print(repr(err))
|
|
error_list.append([filename, repr(err)])
|
|
|
|
#print('Regenerate FIRe done')
|
|
|
|
PATH = './dataset/PhycoCTD/raw'
|
|
for filename in os.listdir(PATH):
|
|
try:
|
|
probeutils.check_if_old_phyco(filename)
|
|
probeutils.execution_pb_phyco(filename)
|
|
except Exception as err:
|
|
#print(filename)
|
|
#print(repr(err))
|
|
error_list.append([filename, repr(err)])
|
|
|
|
#print('Regenerate Phyco done')
|
|
|
|
return render_template('regenerate.html', error_list=error_list)
|
|
|
|
if __name__ == '__main__':
|
|
app.run()
|