diff --git a/dataset/README.md b/dataset/README.md index bef6593..0ce8bee 100644 --- a/dataset/README.md +++ b/dataset/README.md @@ -1,2 +1 @@ -This folder contains all the sensor data uploaded to the server - +Barcode generator \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index ca58d4f..c426290 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,26 +2,11 @@ version: '3.7' services: - ## TODO: Add true proxy WSGI Server instead of running the builtin - uploadtool: + ean13generator: build: . restart: unless-stopped - volumes: - - /mnt/storage/dataset-neo:/app/dataset labels: - traefik.enable=true - - traefik.http.routers.sdc-uploader.entryPoints=web-secure - - traefik.http.routers.sdc-uploader.rule=Host(`upload.med.upct.es`) - - traefik.http.routers.sdc-uploader.tls.certresolver=default - - download: - image: abiosoft/caddy - restart: unless-stopped - volumes: - - /mnt/storage/dataset-neo:/srv - labels: - - traefik.enable=true - - traefik.http.routers.sdc-download.entryPoints=web-secure - - traefik.http.routers.sdc-download.rule=Host(`download.med.upct.es`) - - traefik.http.routers.sdc-download.tls.certresolver=default - - traefik.http.services.sdc-download.loadbalancer.server.port=2015 \ No newline at end of file + - traefik.http.routers.ean13generator.entryPoints=web-secure + - traefik.http.routers.ean13generator.rule=Host(`ean13.fosc.space`) + - traefik.http.routers.ean13generator.tls.certresolver=default diff --git a/main.py b/main.py index 50a99ec..3fa952f 100644 --- a/main.py +++ b/main.py @@ -1,12 +1,10 @@ from flask import Flask, render_template, request from datetime import datetime -from probeutils import utils as probeutils import re import fnmatch import os import meinheld - app = Flask(__name__) app.config['UPLOAD_FOLDER'] = "./dataset/" @@ -14,41 +12,10 @@ app.config['UPLOAD_FOLDER'] = "./dataset/" app.config['MAX_CONTENT_LENGTH'] = 10000000000 # 10GB meinheld.set_max_content_length(100*1024*1024) -app.config['DOWNLOADS_URL'] = "https://download.med.upct.es/" - -probes = [{ - 'sensor': 'SUNA', - 'img': 'suna.jpg', - 'active': ['SATSLF0037'], - 'stations': ['M1', 'M2', 'M3'] -}, { - 'sensor': 'FIRe', - 'img': 'fire.jpg', - 'active': ['SATFIS0006'], - 'stations': ['M1', 'M2', 'M3'] -}, { - 'sensor': 'PhycoCTD', - 'img': 'phycoctd.jpg', - 'active': ['phyco_v1', 'phyco_v2'], - 'stations': ['M1', 'M2', 'M3'] -}, { - 'sensor': 'CastAway', - 'img': 'castaway.jpg', - 'active': ['CC1326008'], - 'stations': ['M1', 'M2', 'M3'] -} -#,{ -# 'sensor': 'GoPro', -# 'img': 'gopro.png', -# 'active': ['gopro1'], -# 'stations': ['M1', 'M2', 'M3'] -#} -] - # Return our beautiful Bootstrap webpage. That we totally have. @app.route('/') def upload(): - return render_template('upload.html', probes=probes) + return render_template('upload.html') # What happens when the files just don't fit @app.errorhandler(413) @@ -66,131 +33,13 @@ def upload_file(): if request.method == 'POST': f = request.files['file'] - # Get which probe the user uploaded - probe = request.form.get('probe') - - # Get selected probe - activeProbe = request.form.get('activeProbe') - - # Check if its a valid file to upload - # TODO: Implement this method - # probeutils.check(probe, f) - - # Check if folder exists - try: - if not os.path.exists( - os.path.join(app.config['UPLOAD_FOLDER'], probe)): - os.makedirs(os.path.join(app.config['UPLOAD_FOLDER'], probe)) - - if not os.path.exists( - os.path.join(app.config['UPLOAD_FOLDER'], probe, 'raw')): - os.makedirs(os.path.join( - app.config['UPLOAD_FOLDER'], probe, 'raw')) - except Exception: - return render_template('error.html') - - # Strip station - # if forceStation is checked, override all station parsing - if (request.form.get("forceStation" + probe) != None): - station = request.form.get("stations" + probe) - else: - if (fnmatch.fnmatch((f.filename).upper(), '*M1*') or fnmatch.fnmatch((f.filename).upper(), 'M1*')): - station = 'M1' - elif (fnmatch.fnmatch((f.filename).upper(), "*M2*") or fnmatch.fnmatch((f.filename).upper(), "M2*")): - station = 'M2' - elif (fnmatch.fnmatch((f.filename).upper(), "*M3*") or fnmatch.fnmatch((f.filename).upper(), "M3*")): - station = 'M3' - - # Date Parser of filename - match1 = re.search('\d{4}-\d{2}-\d{2}', f.filename) - if (match1 == None): - match1 = re.search('\d{2}-\d{2}-\d{4}', f.filename) - try: - date = datetime.strptime(match1.group(), '%d-%m-%Y').date() - except Exception: - # Error! More strange data format XD - match1 = re.search('\d{4}\d{2}\d{2}', f.filename) - if (match1 == None): - match1 = re.search('\d{2}\d{2}\d{4}', f.filename) - date = datetime.strptime(match1.group(), '%d%m%Y').date() - else: - date = datetime.strptime(match1.group(), '%Y%m%d').date() - else: - date = datetime.strptime(match1.group(), '%Y-%m-%d').date() - - # Crafting the real overpowered name!!!! - filename = probe + '-' + station + '-'+ activeProbe + '-'+ date.strftime("%Y-%m-%d") + '.csv' - #print(filename) - - # Setting the filePath and saving it - finalFilename = 'raw-'+filename - rawFilePath = os.path.join(app.config['UPLOAD_FOLDER'], probe, 'raw', finalFilename) - f.save(rawFilePath) - - # Start with probe definition - if (probe == 'CastAway'): - ''' CastAway Files ''' - - df_table = probeutils.process_castaway(rawFilePath) - - elif (probe == 'SUNA'): - ''' SUNA Files ''' - - df_table = probeutils.process_suna(rawFilePath) - - elif (probe == 'FIRe'): - ''' FIRe Files ''' - - df_table = probeutils.process_fire(rawFilePath) - - elif (probe == 'PhycoCTD'): - ''' PhycoCTD Files ''' - - df_table = probeutils.process_phyco(rawFilePath) - else: - ''' Empty probe ''' - - df_table = "
" - - # Save file on raw folder and create URL download - url_download = app.config['DOWNLOADS_URL'] + probe + "/" + finalFilename + # Return success webpage - return render_template('successful.html', url=url_download, tables=df_table) + return render_template('successful.html') else: # No POST Found return render_template('error.html') -## Force regenerate QC -@app.route('/regenerate') -def regenerate(): - error_list = [] - #for each file in raw folder, execute all QC - PATH = './dataset/FIRe/raw' - for filename in os.listdir(PATH): - try: - filename_nn = probeutils.execution_NN(filename) - probeutils.execution_PAR(filename_nn) - except Exception as err: - #print(filename) - #print(repr(err)) - error_list.append([filename, repr(err)]) - - #print('Regenerate FIRe done') - - PATH = './dataset/PhycoCTD/raw' - for filename in os.listdir(PATH): - try: - probeutils.check_if_old_phyco(filename) - probeutils.execution_pb_phyco(filename) - except Exception as err: - #print(filename) - #print(repr(err)) - error_list.append([filename, repr(err)]) - - #print('Regenerate Phyco done') - - return render_template('regenerate.html', error_list=error_list) - if __name__ == '__main__': app.run() diff --git a/probeutils/utils.py b/probeutils/utils.py deleted file mode 100644 index dd352e8..0000000 --- a/probeutils/utils.py +++ /dev/null @@ -1,293 +0,0 @@ -from datetime import datetime -from shutil import copy2 -import pandas as pd -import numpy as np -import os, csv - -UPLOAD_FOLDER = './dataset/' -tableClass = 'table table-striped table-sm table-responsive' - -def get_date(probe, file): - ''' Test utils file ''' - return datetime.now().strftime("%Y-%m-%d-%H:%M:%S") - -# TODO: -# - Check if is a good file -# -#def checkUploadedFile(probe, file): -# return 1 - -# TODO: -# - clean the head of csv, need to extract info an remove the %% (maybe new file with metadata info?) [done] -# - create the empty csv, first row with headers [done] -# - desviation and more data things -def process_castaway(file): - - try: - # Copy raw file to process folder - with open(file) as CastAwayFile: - filename = os.path.basename(CastAwayFile.name).strip('raw-') - processFilePath = os.path.join( - UPLOAD_FOLDER, 'CastAway', filename) - #copy2(file, processFilePath) - - # Getting some metadata info - #with open(file, newline='') as castfile: - # lines = castfile.readlines() - # device = lines[0].split(',')[1].replace('\r\n', '') - # filename = lines[1].split(',')[1].replace('\r\n', '') - # start_latitude = lines[9].split(',')[1].replace('\r\n', '') - # start_longitude = lines[10].split(',')[1].replace('\r\n', '') - # start_altitude = lines[11].split(',')[1].replace('\r\n', '') - # castfile.close() - - # Opening the csv with pandas - df = pd.read_csv(file, skiprows=28) - - # Extract perfil bajada, el 1 es por el header - index_max = df['Depth (Meter)'].idxmax() + 1 - #df_perfilBajada = df[df['depth'].between(0, df['depth'].max())] # No funciona muy bien - # Limitamos a solo el perfil de bajada - df_perfilBajada = df[:index_max] - - # Guardamos solo el perfilBajada en carpeta PB - if not os.path.exists(os.path.join(UPLOAD_FOLDER, 'CastAway', 'PB')): - os.makedirs(os.path.join(UPLOAD_FOLDER, 'CastAway', 'PB')) - - filenamePB = filename.strip('.csv') + '-PB.csv' - perfilBajadaFilePath = os.path.join( - UPLOAD_FOLDER, 'CastAway', 'PB', filenamePB) - df_perfilBajada.to_csv(perfilBajadaFilePath, index=False) - - # Trying to show Dataframe on webpage - return df.to_html(classes=tableClass) - - except Exception as ex: - print('Exception: '+repr(ex)) - return ex - -def process_suna(file): - try: - df = pd.read_csv(file, encoding="ISO-8859-1", header=None) - - #df.columns = ['fechaHora', 'INSTRUMENT', 'Start-time', 'Nitrato(uMol/L)','Nitrato(MG/L)', 'ERROR', 'T_lamp', ] - - #if not os.path.exists(os.path.join(UPLOAD_FOLDER, 'SUNA', 'HEAD')): - # os.makedirs(os.path.join(UPLOAD_FOLDER, 'SUNA', 'HEAD')) - - # Return webpage - return df.to_html(classes=tableClass) - except Exception as ex: - print('Exception: '+repr(ex)) - return ex - -# TODO: -# - Ask for FIRe examples, actually only bin files found (done) -# - Extract file to process folder (done) -# - Save a File with PAR info -# - Headers on email (ask for new headers)[done] -# - Remove negative values (this, done) -# - Arrange similar windows depth and do measure -def process_fire(file): - ''' Processing FIRe ''' - - try: - # Copy raw file to process folder - with open(file) as FIReFile: - filename_raw = os.path.basename(FIReFile.name) - filename = os.path.basename(FIReFile.name).strip('raw-') - #raw_file_path = os.path.join(UPLOAD_FOLDER, 'FIRe', 'raw', filename_raw) - processFilePath = os.path.join( - UPLOAD_FOLDER, 'FIRe', filename) - #copy2(file, processFilePath) - - # First, need to check the csv headers - # Open the process file with pandas - df = pd.read_csv(file, header=None) - - # Headers (now working fine) - df.columns = ['fechaHora', 'estacion', 'fecha', 'hora', 'profundidad', 'Fo', 'Fm', 'Fv', 'Fv/Fm', 'p', 'Abs_rel', 'Abs_abs', 'led_light', 'ETR', 'coma1', 'coma2', 'coma3', 'coma4', 'coma5', 'coma6','error_norm', 'PAR', 'V', 'cero1', 'cero2', 'cero3', 'cero4', 'raro'] - df.loc[:, 'coma1'] = 0 - df.loc[:, 'coma2'] = 0 - df.loc[:, 'coma3'] = 0 - df.loc[:, 'coma4'] = 0 - df.loc[:, 'coma5'] = 0 - df.loc[:, 'coma6'] = 0 - - # Fixing empty values - #df.to_csv(os.path.join(UPLOAD_FOLDER, 'FIRe', 'raw', 'raw-'+filename), index=False) - - filename_nn = execution_NN(filename_raw) - - execution_PAR(filename_nn) - - return df.to_html(classes=tableClass) - - except Exception as ex: - print('Exception: '+repr(ex)) - return ex - -# TODO: -# - Only perfil bajada (done) -def process_phyco(file): - ''' Processing PhycoCTD''' - - try: - # Copy raw file to process folder - with open(file, "r+") as phycoFile: - # Sustract raw- of filename - filename_raw = os.path.basename(phycoFile.name) - filename = os.path.basename(phycoFile.name).strip('raw-') - processFilePath = os.path.join( - UPLOAD_FOLDER, 'PhycoCTD', filename) - #copy2(file, processFilePath) - - # Working with the process file - df = pd.read_csv(file, delimiter=';') - - check_if_old_phyco(filename_raw) - execution_pb_phyco(filename_raw) - - # Return webpage - return df.to_html(classes=tableClass) - except Exception as ex: - print('Exception: '+repr(ex)) - return ex - -def check_if_old_phyco(filename): - # Checking if old file csv - PATH = os.path.join(UPLOAD_FOLDER, 'PhycoCTD', 'raw', filename) - with open(PATH, "r+") as phycoFile: - # Check if old version - line = phycoFile.readline() - if (',' in line): - old = True - else: - old = False - - if (old == True): - df_old = pd.read_csv(PATH, header=None, skiprows=1) - if (len(df_old.columns) == 16): - df_old.columns = ['station', 'latitude', 'longitude', 'time', 'depth', 'temp1', 'temp2', 'cdom[gain]', - 'cdom[ppb]', 'cdom[mv]', 'pe[gain]', 'pe[ppb]', 'pe[mv]', 'chl[gain]', 'chl[ppb]', 'chl[mv]'] - else: - df_old.columns = ['station', 'time', 'depth', 'temp1', 'temp2', 'cdom[gain]', 'cdom[ppb]', - 'cdom[mv]', 'pe[gain]', 'pe[ppb]', 'pe[mv]', 'chl[gain]', 'chl[ppb]', 'chl[mv]'] - raw_path = os.path.join(UPLOAD_FOLDER, 'PhycoCTD', 'raw', filename) - df_old.to_csv(raw_path, index=False, sep=';') - -def execution_pb_phyco(filename): - # Guardamos solo el perfilBajada - if not os.path.exists(os.path.join(UPLOAD_FOLDER, 'PhycoCTD', 'PB')): - os.makedirs(os.path.join(UPLOAD_FOLDER, 'PhycoCTD', 'PB')) - - PATH = os.path.join(UPLOAD_FOLDER, 'PhycoCTD', 'raw', filename) - df = pd.read_csv(PATH, delimiter=';') - - # Checking if NaN values exists [for hack lab version csv upload] - if (df['temp2'].isnull().sum() > 0): - df.loc[:, 'temp2'] = 0 - df = df.dropna() - - # Extract perfil bajada, el 1 es por el header - index_max = df['depth'].idxmax() + 1 - #df_perfilBajada = df[df['depth'].between(0, df['depth'].max())] # No funciona muy bien - # Limitamos a solo el perfil de bajada - df_pb= df[:index_max] - - filename_pb = filename.strip('raw-').strip('.csv') + '-PB.csv' - pb_path = os.path.join(UPLOAD_FOLDER, 'PhycoCTD', 'PB', filename_pb) - df_pb.to_csv(pb_path, sep=';', index=False) - -def execution_NN(filename): - - PATH = os.path.join(UPLOAD_FOLDER, 'FIRe', 'raw', filename) - df = pd.read_csv(PATH, header=None) - - # Headers (now working fine) - df.columns = ['fechaHora', 'estacion', 'fecha', 'hora', 'profundidad', 'Fo', 'Fm', 'Fv', 'Fv/Fm', 'p', 'Abs_rel', 'Abs_abs', 'led_light', - 'ETR', 'coma1', 'coma2', 'coma3', 'coma4', 'coma5', 'coma6', 'error_norm', 'PAR', 'V', 'cero1', 'cero2', 'cero3', 'cero4', 'raro'] - df.loc[:, 'coma1'] = 0 - df.loc[:, 'coma2'] = 0 - df.loc[:, 'coma3'] = 0 - df.loc[:, 'coma4'] = 0 - df.loc[:, 'coma5'] = 0 - df.loc[:, 'coma6'] = 0 - - # NonNegative values rutine - if not os.path.exists(os.path.join(UPLOAD_FOLDER, 'FIRe', 'NN')): - os.makedirs(os.path.join(UPLOAD_FOLDER, 'FIRe', 'NN')) - - # Remove rows with negatives values - df_nn = df[(df.iloc[:, 4:27] >= 0).all(1)] - filename_nn = filename.strip('raw-').strip('.csv') + '-NN.csv' - nonnegative_path = os.path.join(UPLOAD_FOLDER, 'FIRe', 'NN', filename_nn) - df_nn.to_csv(nonnegative_path, index=False) - - return filename_nn - -def execution_PAR(filename): - # PAR rutine - if not os.path.exists(os.path.join(UPLOAD_FOLDER, 'FIRe', 'PAR')): - os.makedirs(os.path.join(UPLOAD_FOLDER, 'FIRe', 'PAR')) - - PATH = os.path.join(UPLOAD_FOLDER, 'FIRe', 'NN', filename) - df = pd.read_csv(PATH) - - # PAR execution - PAR_columns = ['estacion', 'fecha', 'profundidad', 'Fo', 'Fm', 'Fv', 'Fv/Fm', 'p', 'Abs_rel', 'Abs_abs', 'led_light', - 'ETR', 'error_norm', 'PAR'] - - df_PAR = pd.DataFrame(columns=PAR_columns) - - index_max = df['profundidad'].idxmax() - index_min = df['profundidad'].idxmin() + 1 - depth_list = df[index_max:index_min]['profundidad'].to_list() - - last_value = df['profundidad'].max() - similar_depth = [] - for i in range(len(depth_list)): - #print(depth_list[i]) - - if (abs(last_value - depth_list[i]) <= 2000): - last_value = depth_list[i] - similar_depth.append(depth_list[i]) - elif (abs(last_value - depth_list[i]) >= 6000): - ## Save the actual list to DataFrame - #print(similar_depth) - index_first = pd.to_numeric(df.index[df['profundidad'] == similar_depth[0]])[0] - index_last = pd.to_numeric(df.index[df['profundidad'] == similar_depth[-1]])[0] + 1 - df_range = df.iloc[index_first:index_last] - - # Working with df_range - estacion = df['estacion'][0] - fecha = df['fecha'][0] - profundidad = df_range['profundidad'].mean() - Fo = df_range['Fo'].mean() - Fm = df_range['Fm'].mean() - Fv = df_range['Fv'].mean() - FvFm = df_range['Fv/Fm'].mean() - p = df_range['p'].mean() - Abs_rel = df_range['Abs_rel'].mean() - Abs_abs = df_range['Abs_abs'].mean() - led_light = df_range['led_light'].mean() - ETR = df_range['ETR'].mean() - error_norm = df_range['error_norm'].mean() - PAR = df_range['PAR'].mean() - data = [estacion, fecha, profundidad, Fo, Fm, Fv, FvFm, p, Abs_rel, Abs_abs, led_light, ETR, error_norm, PAR] - row = pd.Series(data, index=PAR_columns) - df_PAR = df_PAR.append(row, ignore_index=True) - - ## Empty the list - similar_depth = [] - last_value = depth_list[i] - - ## Adding to the new list - similar_depth.append(depth_list[i]) - - # Saving the DataFrame - # TODO: check last values, around 400 depth - filenamePAR = filename.strip('-NN.csv') + '-PAR.csv' - PARFilePath = os.path.join(UPLOAD_FOLDER, 'FIRe', 'PAR', filenamePAR) - df_PAR.to_csv(PARFilePath, index=False) - diff --git a/requirements.txt b/requirements.txt index 508dae9..ef1bd05 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ Flask pandas -meinheld \ No newline at end of file +meinheld +python-barcode \ No newline at end of file diff --git a/static/img/castaway.jpg b/static/img/castaway.jpg deleted file mode 100644 index 8a443dc..0000000 Binary files a/static/img/castaway.jpg and /dev/null differ diff --git a/static/img/fire.jpg b/static/img/fire.jpg deleted file mode 100644 index b41d0b5..0000000 Binary files a/static/img/fire.jpg and /dev/null differ diff --git a/static/img/gopro.png b/static/img/gopro.png deleted file mode 100644 index 3a21c52..0000000 Binary files a/static/img/gopro.png and /dev/null differ diff --git a/static/img/phycoctd.jpg b/static/img/phycoctd.jpg deleted file mode 100644 index 1ca6cb6..0000000 Binary files a/static/img/phycoctd.jpg and /dev/null differ diff --git a/static/img/placeholder.svg b/static/img/placeholder.svg deleted file mode 100644 index f446121..0000000 --- a/static/img/placeholder.svg +++ /dev/null @@ -1 +0,0 @@ -200x200 \ No newline at end of file diff --git a/static/img/suna.jpg b/static/img/suna.jpg deleted file mode 100644 index fe31850..0000000 Binary files a/static/img/suna.jpg and /dev/null differ diff --git a/templates/upload.html b/templates/upload.html index ce4ac8d..b5359b1 100644 --- a/templates/upload.html +++ b/templates/upload.html @@ -1,127 +1,78 @@ -{% extends "base.html" %} -{% block title %}Subida{% endblock %} - -{% block content %} +{% extends "base.html" %} {% block title %}Subida{% endblock %} {% block content %}
-

Sensores

+

Upload

-
- - -
- {% for i in probes %} -
-
-
-

{{ i.sensor }}

-
- -
-
- ... -
-
- - -
+
+ +
+
- {% endfor%} + + +
+ +
- - {% for i in probes %} -
- {% endfor %} - - -
+ + + + +
+ {% endblock %} \ No newline at end of file