This commit is contained in:
parent
6cdf4d5197
commit
cadecba59a
@ -1,2 +1 @@
|
|||||||
This folder contains all the sensor data uploaded to the server
|
Barcode generator
|
||||||
|
|
@ -2,26 +2,11 @@ version: '3.7'
|
|||||||
|
|
||||||
services:
|
services:
|
||||||
|
|
||||||
## TODO: Add true proxy WSGI Server instead of running the builtin
|
ean13generator:
|
||||||
uploadtool:
|
|
||||||
build: .
|
build: .
|
||||||
restart: unless-stopped
|
restart: unless-stopped
|
||||||
volumes:
|
|
||||||
- /mnt/storage/dataset-neo:/app/dataset
|
|
||||||
labels:
|
labels:
|
||||||
- traefik.enable=true
|
- traefik.enable=true
|
||||||
- traefik.http.routers.sdc-uploader.entryPoints=web-secure
|
- traefik.http.routers.ean13generator.entryPoints=web-secure
|
||||||
- traefik.http.routers.sdc-uploader.rule=Host(`upload.med.upct.es`)
|
- traefik.http.routers.ean13generator.rule=Host(`ean13.fosc.space`)
|
||||||
- traefik.http.routers.sdc-uploader.tls.certresolver=default
|
- traefik.http.routers.ean13generator.tls.certresolver=default
|
||||||
|
|
||||||
download:
|
|
||||||
image: abiosoft/caddy
|
|
||||||
restart: unless-stopped
|
|
||||||
volumes:
|
|
||||||
- /mnt/storage/dataset-neo:/srv
|
|
||||||
labels:
|
|
||||||
- traefik.enable=true
|
|
||||||
- traefik.http.routers.sdc-download.entryPoints=web-secure
|
|
||||||
- traefik.http.routers.sdc-download.rule=Host(`download.med.upct.es`)
|
|
||||||
- traefik.http.routers.sdc-download.tls.certresolver=default
|
|
||||||
- traefik.http.services.sdc-download.loadbalancer.server.port=2015
|
|
||||||
|
155
main.py
155
main.py
@ -1,12 +1,10 @@
|
|||||||
from flask import Flask, render_template, request
|
from flask import Flask, render_template, request
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
from probeutils import utils as probeutils
|
|
||||||
import re
|
import re
|
||||||
import fnmatch
|
import fnmatch
|
||||||
import os
|
import os
|
||||||
import meinheld
|
import meinheld
|
||||||
|
|
||||||
|
|
||||||
app = Flask(__name__)
|
app = Flask(__name__)
|
||||||
|
|
||||||
app.config['UPLOAD_FOLDER'] = "./dataset/"
|
app.config['UPLOAD_FOLDER'] = "./dataset/"
|
||||||
@ -14,41 +12,10 @@ app.config['UPLOAD_FOLDER'] = "./dataset/"
|
|||||||
app.config['MAX_CONTENT_LENGTH'] = 10000000000 # 10GB
|
app.config['MAX_CONTENT_LENGTH'] = 10000000000 # 10GB
|
||||||
meinheld.set_max_content_length(100*1024*1024)
|
meinheld.set_max_content_length(100*1024*1024)
|
||||||
|
|
||||||
app.config['DOWNLOADS_URL'] = "https://download.med.upct.es/"
|
|
||||||
|
|
||||||
probes = [{
|
|
||||||
'sensor': 'SUNA',
|
|
||||||
'img': 'suna.jpg',
|
|
||||||
'active': ['SATSLF0037'],
|
|
||||||
'stations': ['M1', 'M2', 'M3']
|
|
||||||
}, {
|
|
||||||
'sensor': 'FIRe',
|
|
||||||
'img': 'fire.jpg',
|
|
||||||
'active': ['SATFIS0006'],
|
|
||||||
'stations': ['M1', 'M2', 'M3']
|
|
||||||
}, {
|
|
||||||
'sensor': 'PhycoCTD',
|
|
||||||
'img': 'phycoctd.jpg',
|
|
||||||
'active': ['phyco_v1', 'phyco_v2'],
|
|
||||||
'stations': ['M1', 'M2', 'M3']
|
|
||||||
}, {
|
|
||||||
'sensor': 'CastAway',
|
|
||||||
'img': 'castaway.jpg',
|
|
||||||
'active': ['CC1326008'],
|
|
||||||
'stations': ['M1', 'M2', 'M3']
|
|
||||||
}
|
|
||||||
#,{
|
|
||||||
# 'sensor': 'GoPro',
|
|
||||||
# 'img': 'gopro.png',
|
|
||||||
# 'active': ['gopro1'],
|
|
||||||
# 'stations': ['M1', 'M2', 'M3']
|
|
||||||
#}
|
|
||||||
]
|
|
||||||
|
|
||||||
# Return our beautiful Bootstrap webpage. That we totally have.
|
# Return our beautiful Bootstrap webpage. That we totally have.
|
||||||
@app.route('/')
|
@app.route('/')
|
||||||
def upload():
|
def upload():
|
||||||
return render_template('upload.html', probes=probes)
|
return render_template('upload.html')
|
||||||
|
|
||||||
# What happens when the files just don't fit
|
# What happens when the files just don't fit
|
||||||
@app.errorhandler(413)
|
@app.errorhandler(413)
|
||||||
@ -66,131 +33,13 @@ def upload_file():
|
|||||||
if request.method == 'POST':
|
if request.method == 'POST':
|
||||||
f = request.files['file']
|
f = request.files['file']
|
||||||
|
|
||||||
# Get which probe the user uploaded
|
|
||||||
probe = request.form.get('probe')
|
|
||||||
|
|
||||||
# Get selected probe
|
|
||||||
activeProbe = request.form.get('activeProbe')
|
|
||||||
|
|
||||||
# Check if its a valid file to upload
|
|
||||||
# TODO: Implement this method
|
|
||||||
# probeutils.check(probe, f)
|
|
||||||
|
|
||||||
# Check if folder exists
|
|
||||||
try:
|
|
||||||
if not os.path.exists(
|
|
||||||
os.path.join(app.config['UPLOAD_FOLDER'], probe)):
|
|
||||||
os.makedirs(os.path.join(app.config['UPLOAD_FOLDER'], probe))
|
|
||||||
|
|
||||||
if not os.path.exists(
|
|
||||||
os.path.join(app.config['UPLOAD_FOLDER'], probe, 'raw')):
|
|
||||||
os.makedirs(os.path.join(
|
|
||||||
app.config['UPLOAD_FOLDER'], probe, 'raw'))
|
|
||||||
except Exception:
|
|
||||||
return render_template('error.html')
|
|
||||||
|
|
||||||
# Strip station
|
|
||||||
# if forceStation is checked, override all station parsing
|
|
||||||
if (request.form.get("forceStation" + probe) != None):
|
|
||||||
station = request.form.get("stations" + probe)
|
|
||||||
else:
|
|
||||||
if (fnmatch.fnmatch((f.filename).upper(), '*M1*') or fnmatch.fnmatch((f.filename).upper(), 'M1*')):
|
|
||||||
station = 'M1'
|
|
||||||
elif (fnmatch.fnmatch((f.filename).upper(), "*M2*") or fnmatch.fnmatch((f.filename).upper(), "M2*")):
|
|
||||||
station = 'M2'
|
|
||||||
elif (fnmatch.fnmatch((f.filename).upper(), "*M3*") or fnmatch.fnmatch((f.filename).upper(), "M3*")):
|
|
||||||
station = 'M3'
|
|
||||||
|
|
||||||
# Date Parser of filename
|
|
||||||
match1 = re.search('\d{4}-\d{2}-\d{2}', f.filename)
|
|
||||||
if (match1 == None):
|
|
||||||
match1 = re.search('\d{2}-\d{2}-\d{4}', f.filename)
|
|
||||||
try:
|
|
||||||
date = datetime.strptime(match1.group(), '%d-%m-%Y').date()
|
|
||||||
except Exception:
|
|
||||||
# Error! More strange data format XD
|
|
||||||
match1 = re.search('\d{4}\d{2}\d{2}', f.filename)
|
|
||||||
if (match1 == None):
|
|
||||||
match1 = re.search('\d{2}\d{2}\d{4}', f.filename)
|
|
||||||
date = datetime.strptime(match1.group(), '%d%m%Y').date()
|
|
||||||
else:
|
|
||||||
date = datetime.strptime(match1.group(), '%Y%m%d').date()
|
|
||||||
else:
|
|
||||||
date = datetime.strptime(match1.group(), '%Y-%m-%d').date()
|
|
||||||
|
|
||||||
# Crafting the real overpowered name!!!!
|
|
||||||
filename = probe + '-' + station + '-'+ activeProbe + '-'+ date.strftime("%Y-%m-%d") + '.csv'
|
|
||||||
#print(filename)
|
|
||||||
|
|
||||||
# Setting the filePath and saving it
|
|
||||||
finalFilename = 'raw-'+filename
|
|
||||||
rawFilePath = os.path.join(app.config['UPLOAD_FOLDER'], probe, 'raw', finalFilename)
|
|
||||||
f.save(rawFilePath)
|
|
||||||
|
|
||||||
# Start with probe definition
|
|
||||||
if (probe == 'CastAway'):
|
|
||||||
''' CastAway Files '''
|
|
||||||
|
|
||||||
df_table = probeutils.process_castaway(rawFilePath)
|
|
||||||
|
|
||||||
elif (probe == 'SUNA'):
|
|
||||||
''' SUNA Files '''
|
|
||||||
|
|
||||||
df_table = probeutils.process_suna(rawFilePath)
|
|
||||||
|
|
||||||
elif (probe == 'FIRe'):
|
|
||||||
''' FIRe Files '''
|
|
||||||
|
|
||||||
df_table = probeutils.process_fire(rawFilePath)
|
|
||||||
|
|
||||||
elif (probe == 'PhycoCTD'):
|
|
||||||
''' PhycoCTD Files '''
|
|
||||||
|
|
||||||
df_table = probeutils.process_phyco(rawFilePath)
|
|
||||||
else:
|
|
||||||
''' Empty probe '''
|
|
||||||
|
|
||||||
df_table = "<div></div>"
|
|
||||||
|
|
||||||
# Save file on raw folder and create URL download
|
|
||||||
url_download = app.config['DOWNLOADS_URL'] + probe + "/" + finalFilename
|
|
||||||
|
|
||||||
# Return success webpage
|
# Return success webpage
|
||||||
return render_template('successful.html', url=url_download, tables=df_table)
|
return render_template('successful.html')
|
||||||
else:
|
else:
|
||||||
# No POST Found
|
# No POST Found
|
||||||
return render_template('error.html')
|
return render_template('error.html')
|
||||||
|
|
||||||
## Force regenerate QC
|
|
||||||
@app.route('/regenerate')
|
|
||||||
def regenerate():
|
|
||||||
error_list = []
|
|
||||||
#for each file in raw folder, execute all QC
|
|
||||||
PATH = './dataset/FIRe/raw'
|
|
||||||
for filename in os.listdir(PATH):
|
|
||||||
try:
|
|
||||||
filename_nn = probeutils.execution_NN(filename)
|
|
||||||
probeutils.execution_PAR(filename_nn)
|
|
||||||
except Exception as err:
|
|
||||||
#print(filename)
|
|
||||||
#print(repr(err))
|
|
||||||
error_list.append([filename, repr(err)])
|
|
||||||
|
|
||||||
#print('Regenerate FIRe done')
|
|
||||||
|
|
||||||
PATH = './dataset/PhycoCTD/raw'
|
|
||||||
for filename in os.listdir(PATH):
|
|
||||||
try:
|
|
||||||
probeutils.check_if_old_phyco(filename)
|
|
||||||
probeutils.execution_pb_phyco(filename)
|
|
||||||
except Exception as err:
|
|
||||||
#print(filename)
|
|
||||||
#print(repr(err))
|
|
||||||
error_list.append([filename, repr(err)])
|
|
||||||
|
|
||||||
#print('Regenerate Phyco done')
|
|
||||||
|
|
||||||
return render_template('regenerate.html', error_list=error_list)
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
if __name__ == '__main__':
|
||||||
app.run()
|
app.run()
|
||||||
|
@ -1,293 +0,0 @@
|
|||||||
from datetime import datetime
|
|
||||||
from shutil import copy2
|
|
||||||
import pandas as pd
|
|
||||||
import numpy as np
|
|
||||||
import os, csv
|
|
||||||
|
|
||||||
UPLOAD_FOLDER = './dataset/'
|
|
||||||
tableClass = 'table table-striped table-sm table-responsive'
|
|
||||||
|
|
||||||
def get_date(probe, file):
|
|
||||||
''' Test utils file '''
|
|
||||||
return datetime.now().strftime("%Y-%m-%d-%H:%M:%S")
|
|
||||||
|
|
||||||
# TODO:
|
|
||||||
# - Check if is a good file
|
|
||||||
#
|
|
||||||
#def checkUploadedFile(probe, file):
|
|
||||||
# return 1
|
|
||||||
|
|
||||||
# TODO:
|
|
||||||
# - clean the head of csv, need to extract info an remove the %% (maybe new file with metadata info?) [done]
|
|
||||||
# - create the empty csv, first row with headers [done]
|
|
||||||
# - desviation and more data things
|
|
||||||
def process_castaway(file):
|
|
||||||
|
|
||||||
try:
|
|
||||||
# Copy raw file to process folder
|
|
||||||
with open(file) as CastAwayFile:
|
|
||||||
filename = os.path.basename(CastAwayFile.name).strip('raw-')
|
|
||||||
processFilePath = os.path.join(
|
|
||||||
UPLOAD_FOLDER, 'CastAway', filename)
|
|
||||||
#copy2(file, processFilePath)
|
|
||||||
|
|
||||||
# Getting some metadata info
|
|
||||||
#with open(file, newline='') as castfile:
|
|
||||||
# lines = castfile.readlines()
|
|
||||||
# device = lines[0].split(',')[1].replace('\r\n', '')
|
|
||||||
# filename = lines[1].split(',')[1].replace('\r\n', '')
|
|
||||||
# start_latitude = lines[9].split(',')[1].replace('\r\n', '')
|
|
||||||
# start_longitude = lines[10].split(',')[1].replace('\r\n', '')
|
|
||||||
# start_altitude = lines[11].split(',')[1].replace('\r\n', '')
|
|
||||||
# castfile.close()
|
|
||||||
|
|
||||||
# Opening the csv with pandas
|
|
||||||
df = pd.read_csv(file, skiprows=28)
|
|
||||||
|
|
||||||
# Extract perfil bajada, el 1 es por el header
|
|
||||||
index_max = df['Depth (Meter)'].idxmax() + 1
|
|
||||||
#df_perfilBajada = df[df['depth'].between(0, df['depth'].max())] # No funciona muy bien
|
|
||||||
# Limitamos a solo el perfil de bajada
|
|
||||||
df_perfilBajada = df[:index_max]
|
|
||||||
|
|
||||||
# Guardamos solo el perfilBajada en carpeta PB
|
|
||||||
if not os.path.exists(os.path.join(UPLOAD_FOLDER, 'CastAway', 'PB')):
|
|
||||||
os.makedirs(os.path.join(UPLOAD_FOLDER, 'CastAway', 'PB'))
|
|
||||||
|
|
||||||
filenamePB = filename.strip('.csv') + '-PB.csv'
|
|
||||||
perfilBajadaFilePath = os.path.join(
|
|
||||||
UPLOAD_FOLDER, 'CastAway', 'PB', filenamePB)
|
|
||||||
df_perfilBajada.to_csv(perfilBajadaFilePath, index=False)
|
|
||||||
|
|
||||||
# Trying to show Dataframe on webpage
|
|
||||||
return df.to_html(classes=tableClass)
|
|
||||||
|
|
||||||
except Exception as ex:
|
|
||||||
print('Exception: '+repr(ex))
|
|
||||||
return ex
|
|
||||||
|
|
||||||
def process_suna(file):
|
|
||||||
try:
|
|
||||||
df = pd.read_csv(file, encoding="ISO-8859-1", header=None)
|
|
||||||
|
|
||||||
#df.columns = ['fechaHora', 'INSTRUMENT', 'Start-time', 'Nitrato(uMol/L)','Nitrato(MG/L)', 'ERROR', 'T_lamp', ]
|
|
||||||
|
|
||||||
#if not os.path.exists(os.path.join(UPLOAD_FOLDER, 'SUNA', 'HEAD')):
|
|
||||||
# os.makedirs(os.path.join(UPLOAD_FOLDER, 'SUNA', 'HEAD'))
|
|
||||||
|
|
||||||
# Return webpage
|
|
||||||
return df.to_html(classes=tableClass)
|
|
||||||
except Exception as ex:
|
|
||||||
print('Exception: '+repr(ex))
|
|
||||||
return ex
|
|
||||||
|
|
||||||
# TODO:
|
|
||||||
# - Ask for FIRe examples, actually only bin files found (done)
|
|
||||||
# - Extract file to process folder (done)
|
|
||||||
# - Save a File with PAR info
|
|
||||||
# - Headers on email (ask for new headers)[done]
|
|
||||||
# - Remove negative values (this, done)
|
|
||||||
# - Arrange similar windows depth and do measure
|
|
||||||
def process_fire(file):
|
|
||||||
''' Processing FIRe '''
|
|
||||||
|
|
||||||
try:
|
|
||||||
# Copy raw file to process folder
|
|
||||||
with open(file) as FIReFile:
|
|
||||||
filename_raw = os.path.basename(FIReFile.name)
|
|
||||||
filename = os.path.basename(FIReFile.name).strip('raw-')
|
|
||||||
#raw_file_path = os.path.join(UPLOAD_FOLDER, 'FIRe', 'raw', filename_raw)
|
|
||||||
processFilePath = os.path.join(
|
|
||||||
UPLOAD_FOLDER, 'FIRe', filename)
|
|
||||||
#copy2(file, processFilePath)
|
|
||||||
|
|
||||||
# First, need to check the csv headers
|
|
||||||
# Open the process file with pandas
|
|
||||||
df = pd.read_csv(file, header=None)
|
|
||||||
|
|
||||||
# Headers (now working fine)
|
|
||||||
df.columns = ['fechaHora', 'estacion', 'fecha', 'hora', 'profundidad', 'Fo', 'Fm', 'Fv', 'Fv/Fm', 'p', 'Abs_rel', 'Abs_abs', 'led_light', 'ETR', 'coma1', 'coma2', 'coma3', 'coma4', 'coma5', 'coma6','error_norm', 'PAR', 'V', 'cero1', 'cero2', 'cero3', 'cero4', 'raro']
|
|
||||||
df.loc[:, 'coma1'] = 0
|
|
||||||
df.loc[:, 'coma2'] = 0
|
|
||||||
df.loc[:, 'coma3'] = 0
|
|
||||||
df.loc[:, 'coma4'] = 0
|
|
||||||
df.loc[:, 'coma5'] = 0
|
|
||||||
df.loc[:, 'coma6'] = 0
|
|
||||||
|
|
||||||
# Fixing empty values
|
|
||||||
#df.to_csv(os.path.join(UPLOAD_FOLDER, 'FIRe', 'raw', 'raw-'+filename), index=False)
|
|
||||||
|
|
||||||
filename_nn = execution_NN(filename_raw)
|
|
||||||
|
|
||||||
execution_PAR(filename_nn)
|
|
||||||
|
|
||||||
return df.to_html(classes=tableClass)
|
|
||||||
|
|
||||||
except Exception as ex:
|
|
||||||
print('Exception: '+repr(ex))
|
|
||||||
return ex
|
|
||||||
|
|
||||||
# TODO:
|
|
||||||
# - Only perfil bajada (done)
|
|
||||||
def process_phyco(file):
|
|
||||||
''' Processing PhycoCTD'''
|
|
||||||
|
|
||||||
try:
|
|
||||||
# Copy raw file to process folder
|
|
||||||
with open(file, "r+") as phycoFile:
|
|
||||||
# Sustract raw- of filename
|
|
||||||
filename_raw = os.path.basename(phycoFile.name)
|
|
||||||
filename = os.path.basename(phycoFile.name).strip('raw-')
|
|
||||||
processFilePath = os.path.join(
|
|
||||||
UPLOAD_FOLDER, 'PhycoCTD', filename)
|
|
||||||
#copy2(file, processFilePath)
|
|
||||||
|
|
||||||
# Working with the process file
|
|
||||||
df = pd.read_csv(file, delimiter=';')
|
|
||||||
|
|
||||||
check_if_old_phyco(filename_raw)
|
|
||||||
execution_pb_phyco(filename_raw)
|
|
||||||
|
|
||||||
# Return webpage
|
|
||||||
return df.to_html(classes=tableClass)
|
|
||||||
except Exception as ex:
|
|
||||||
print('Exception: '+repr(ex))
|
|
||||||
return ex
|
|
||||||
|
|
||||||
def check_if_old_phyco(filename):
|
|
||||||
# Checking if old file csv
|
|
||||||
PATH = os.path.join(UPLOAD_FOLDER, 'PhycoCTD', 'raw', filename)
|
|
||||||
with open(PATH, "r+") as phycoFile:
|
|
||||||
# Check if old version
|
|
||||||
line = phycoFile.readline()
|
|
||||||
if (',' in line):
|
|
||||||
old = True
|
|
||||||
else:
|
|
||||||
old = False
|
|
||||||
|
|
||||||
if (old == True):
|
|
||||||
df_old = pd.read_csv(PATH, header=None, skiprows=1)
|
|
||||||
if (len(df_old.columns) == 16):
|
|
||||||
df_old.columns = ['station', 'latitude', 'longitude', 'time', 'depth', 'temp1', 'temp2', 'cdom[gain]',
|
|
||||||
'cdom[ppb]', 'cdom[mv]', 'pe[gain]', 'pe[ppb]', 'pe[mv]', 'chl[gain]', 'chl[ppb]', 'chl[mv]']
|
|
||||||
else:
|
|
||||||
df_old.columns = ['station', 'time', 'depth', 'temp1', 'temp2', 'cdom[gain]', 'cdom[ppb]',
|
|
||||||
'cdom[mv]', 'pe[gain]', 'pe[ppb]', 'pe[mv]', 'chl[gain]', 'chl[ppb]', 'chl[mv]']
|
|
||||||
raw_path = os.path.join(UPLOAD_FOLDER, 'PhycoCTD', 'raw', filename)
|
|
||||||
df_old.to_csv(raw_path, index=False, sep=';')
|
|
||||||
|
|
||||||
def execution_pb_phyco(filename):
|
|
||||||
# Guardamos solo el perfilBajada
|
|
||||||
if not os.path.exists(os.path.join(UPLOAD_FOLDER, 'PhycoCTD', 'PB')):
|
|
||||||
os.makedirs(os.path.join(UPLOAD_FOLDER, 'PhycoCTD', 'PB'))
|
|
||||||
|
|
||||||
PATH = os.path.join(UPLOAD_FOLDER, 'PhycoCTD', 'raw', filename)
|
|
||||||
df = pd.read_csv(PATH, delimiter=';')
|
|
||||||
|
|
||||||
# Checking if NaN values exists [for hack lab version csv upload]
|
|
||||||
if (df['temp2'].isnull().sum() > 0):
|
|
||||||
df.loc[:, 'temp2'] = 0
|
|
||||||
df = df.dropna()
|
|
||||||
|
|
||||||
# Extract perfil bajada, el 1 es por el header
|
|
||||||
index_max = df['depth'].idxmax() + 1
|
|
||||||
#df_perfilBajada = df[df['depth'].between(0, df['depth'].max())] # No funciona muy bien
|
|
||||||
# Limitamos a solo el perfil de bajada
|
|
||||||
df_pb= df[:index_max]
|
|
||||||
|
|
||||||
filename_pb = filename.strip('raw-').strip('.csv') + '-PB.csv'
|
|
||||||
pb_path = os.path.join(UPLOAD_FOLDER, 'PhycoCTD', 'PB', filename_pb)
|
|
||||||
df_pb.to_csv(pb_path, sep=';', index=False)
|
|
||||||
|
|
||||||
def execution_NN(filename):
|
|
||||||
|
|
||||||
PATH = os.path.join(UPLOAD_FOLDER, 'FIRe', 'raw', filename)
|
|
||||||
df = pd.read_csv(PATH, header=None)
|
|
||||||
|
|
||||||
# Headers (now working fine)
|
|
||||||
df.columns = ['fechaHora', 'estacion', 'fecha', 'hora', 'profundidad', 'Fo', 'Fm', 'Fv', 'Fv/Fm', 'p', 'Abs_rel', 'Abs_abs', 'led_light',
|
|
||||||
'ETR', 'coma1', 'coma2', 'coma3', 'coma4', 'coma5', 'coma6', 'error_norm', 'PAR', 'V', 'cero1', 'cero2', 'cero3', 'cero4', 'raro']
|
|
||||||
df.loc[:, 'coma1'] = 0
|
|
||||||
df.loc[:, 'coma2'] = 0
|
|
||||||
df.loc[:, 'coma3'] = 0
|
|
||||||
df.loc[:, 'coma4'] = 0
|
|
||||||
df.loc[:, 'coma5'] = 0
|
|
||||||
df.loc[:, 'coma6'] = 0
|
|
||||||
|
|
||||||
# NonNegative values rutine
|
|
||||||
if not os.path.exists(os.path.join(UPLOAD_FOLDER, 'FIRe', 'NN')):
|
|
||||||
os.makedirs(os.path.join(UPLOAD_FOLDER, 'FIRe', 'NN'))
|
|
||||||
|
|
||||||
# Remove rows with negatives values
|
|
||||||
df_nn = df[(df.iloc[:, 4:27] >= 0).all(1)]
|
|
||||||
filename_nn = filename.strip('raw-').strip('.csv') + '-NN.csv'
|
|
||||||
nonnegative_path = os.path.join(UPLOAD_FOLDER, 'FIRe', 'NN', filename_nn)
|
|
||||||
df_nn.to_csv(nonnegative_path, index=False)
|
|
||||||
|
|
||||||
return filename_nn
|
|
||||||
|
|
||||||
def execution_PAR(filename):
|
|
||||||
# PAR rutine
|
|
||||||
if not os.path.exists(os.path.join(UPLOAD_FOLDER, 'FIRe', 'PAR')):
|
|
||||||
os.makedirs(os.path.join(UPLOAD_FOLDER, 'FIRe', 'PAR'))
|
|
||||||
|
|
||||||
PATH = os.path.join(UPLOAD_FOLDER, 'FIRe', 'NN', filename)
|
|
||||||
df = pd.read_csv(PATH)
|
|
||||||
|
|
||||||
# PAR execution
|
|
||||||
PAR_columns = ['estacion', 'fecha', 'profundidad', 'Fo', 'Fm', 'Fv', 'Fv/Fm', 'p', 'Abs_rel', 'Abs_abs', 'led_light',
|
|
||||||
'ETR', 'error_norm', 'PAR']
|
|
||||||
|
|
||||||
df_PAR = pd.DataFrame(columns=PAR_columns)
|
|
||||||
|
|
||||||
index_max = df['profundidad'].idxmax()
|
|
||||||
index_min = df['profundidad'].idxmin() + 1
|
|
||||||
depth_list = df[index_max:index_min]['profundidad'].to_list()
|
|
||||||
|
|
||||||
last_value = df['profundidad'].max()
|
|
||||||
similar_depth = []
|
|
||||||
for i in range(len(depth_list)):
|
|
||||||
#print(depth_list[i])
|
|
||||||
|
|
||||||
if (abs(last_value - depth_list[i]) <= 2000):
|
|
||||||
last_value = depth_list[i]
|
|
||||||
similar_depth.append(depth_list[i])
|
|
||||||
elif (abs(last_value - depth_list[i]) >= 6000):
|
|
||||||
## Save the actual list to DataFrame
|
|
||||||
#print(similar_depth)
|
|
||||||
index_first = pd.to_numeric(df.index[df['profundidad'] == similar_depth[0]])[0]
|
|
||||||
index_last = pd.to_numeric(df.index[df['profundidad'] == similar_depth[-1]])[0] + 1
|
|
||||||
df_range = df.iloc[index_first:index_last]
|
|
||||||
|
|
||||||
# Working with df_range
|
|
||||||
estacion = df['estacion'][0]
|
|
||||||
fecha = df['fecha'][0]
|
|
||||||
profundidad = df_range['profundidad'].mean()
|
|
||||||
Fo = df_range['Fo'].mean()
|
|
||||||
Fm = df_range['Fm'].mean()
|
|
||||||
Fv = df_range['Fv'].mean()
|
|
||||||
FvFm = df_range['Fv/Fm'].mean()
|
|
||||||
p = df_range['p'].mean()
|
|
||||||
Abs_rel = df_range['Abs_rel'].mean()
|
|
||||||
Abs_abs = df_range['Abs_abs'].mean()
|
|
||||||
led_light = df_range['led_light'].mean()
|
|
||||||
ETR = df_range['ETR'].mean()
|
|
||||||
error_norm = df_range['error_norm'].mean()
|
|
||||||
PAR = df_range['PAR'].mean()
|
|
||||||
data = [estacion, fecha, profundidad, Fo, Fm, Fv, FvFm, p, Abs_rel, Abs_abs, led_light, ETR, error_norm, PAR]
|
|
||||||
row = pd.Series(data, index=PAR_columns)
|
|
||||||
df_PAR = df_PAR.append(row, ignore_index=True)
|
|
||||||
|
|
||||||
## Empty the list
|
|
||||||
similar_depth = []
|
|
||||||
last_value = depth_list[i]
|
|
||||||
|
|
||||||
## Adding to the new list
|
|
||||||
similar_depth.append(depth_list[i])
|
|
||||||
|
|
||||||
# Saving the DataFrame
|
|
||||||
# TODO: check last values, around 400 depth
|
|
||||||
filenamePAR = filename.strip('-NN.csv') + '-PAR.csv'
|
|
||||||
PARFilePath = os.path.join(UPLOAD_FOLDER, 'FIRe', 'PAR', filenamePAR)
|
|
||||||
df_PAR.to_csv(PARFilePath, index=False)
|
|
||||||
|
|
@ -1,3 +1,4 @@
|
|||||||
Flask
|
Flask
|
||||||
pandas
|
pandas
|
||||||
meinheld
|
meinheld
|
||||||
|
python-barcode
|
Binary file not shown.
Before Width: | Height: | Size: 21 KiB |
Binary file not shown.
Before Width: | Height: | Size: 50 KiB |
Binary file not shown.
Before Width: | Height: | Size: 92 KiB |
Binary file not shown.
Before Width: | Height: | Size: 19 KiB |
@ -1 +0,0 @@
|
|||||||
<svg width="200" height="200" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 200 200" preserveAspectRatio="none"><defs><style type="text/css">#holder_171aa8443cc text { fill:rgba(255,255,255,.75);font-weight:normal;font-family:Helvetica, monospace;font-size:10pt } </style></defs><g id="holder_171aa8443cc"><rect width="200" height="200" fill="#777"></rect><g><text x="74.41666603088379" y="104.25000009536743">200x200</text></g></g></svg>
|
|
Before Width: | Height: | Size: 440 B |
Binary file not shown.
Before Width: | Height: | Size: 7.7 KiB |
@ -1,69 +1,12 @@
|
|||||||
{% extends "base.html" %}
|
{% extends "base.html" %} {% block title %}Subida{% endblock %} {% block content %}
|
||||||
{% block title %}Subida{% endblock %}
|
|
||||||
|
|
||||||
{% block content %}
|
|
||||||
|
|
||||||
<!-- Row Header -->
|
<!-- Row Header -->
|
||||||
<div class="px-3 py-3 pt-md-5 pb-md-4 mx-auto text-center">
|
<div class="px-3 py-3 pt-md-5 pb-md-4 mx-auto text-center">
|
||||||
<h1 class="display-4">Sensores</h1>
|
<h1 class="display-4">Upload</h1>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<div class="container">
|
<div class="container">
|
||||||
<div class="card-deck mb-3 text-center"></div>
|
<form class="list-group list-group-flush" action="{{pagename}}/uploader" method="POST" enctype="multipart/form-data">
|
||||||
|
|
||||||
<!-- Sensors row -->
|
|
||||||
<div class="row">
|
|
||||||
{% for i in probes %}
|
|
||||||
<div class="col-sm">
|
|
||||||
<div class="card mb-4 shadow-sm">
|
|
||||||
<div class="card-header">
|
|
||||||
<h4 class="my-0 font-weight-normal">{{ i.sensor }}</h4>
|
|
||||||
</div>
|
|
||||||
|
|
||||||
<div class="card-body">
|
|
||||||
<div class="text-center">
|
|
||||||
<img src="{{ url_for('static',filename='img/'+i.img) }}" class="rounded" alt="..."
|
|
||||||
height="200px" width="200px" />
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
|
|
||||||
<button type="button" class="btn btn-primary" data-toggle="modal" data-target="#upload{{i.sensor}}">
|
|
||||||
Subir datos
|
|
||||||
</button>
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
{% endfor%}
|
|
||||||
</div>
|
|
||||||
|
|
||||||
<!-- Setting up the modal foreach probe -->
|
|
||||||
{% for i in probes %}
|
|
||||||
<div class="modal fade" id="upload{{i.sensor}}" role="dialog">
|
|
||||||
<div class="modal-dialog modal-lg">
|
|
||||||
|
|
||||||
<!-- Modal content-->
|
|
||||||
<div class="modal-content">
|
|
||||||
<div class="modal-header">
|
|
||||||
<h4 class="modal-title">Subir datos de {{ i.sensor }}</h4>
|
|
||||||
|
|
||||||
<button type="button" class="close" data-dismiss="modal">
|
|
||||||
×
|
|
||||||
</button>
|
|
||||||
</div>
|
|
||||||
|
|
||||||
<div class="modal-body">
|
|
||||||
|
|
||||||
<!-- Modal header -->
|
|
||||||
<div class="upload-header px-3 py-3 pt-md-5 pb-md-4 mx-auto text-center">
|
|
||||||
<h2 class="display-5">{{ i.sensor }}</h2>
|
|
||||||
</div>
|
|
||||||
|
|
||||||
<div class="container">
|
|
||||||
<div class="text-center">
|
|
||||||
|
|
||||||
<!-- Modal form -->
|
|
||||||
<form class="list-group list-group-flush" action="{{pagename}}/uploader" method="POST"
|
|
||||||
enctype="multipart/form-data">
|
|
||||||
|
|
||||||
<!-- File input -->
|
<!-- File input -->
|
||||||
<div class="list-group-item" style="padding-left: 50px;">
|
<div class="list-group-item" style="padding-left: 50px;">
|
||||||
<input type="file" class="form-control-file" name="file" />
|
<input type="file" class="form-control-file" name="file" />
|
||||||
@ -71,8 +14,8 @@
|
|||||||
|
|
||||||
<!-- Sensor select -->
|
<!-- Sensor select -->
|
||||||
<div class="form-group required" style="padding-top: 10px;">
|
<div class="form-group required" style="padding-top: 10px;">
|
||||||
<label class="form-check-label" for="activeProbe" style="padding-left: 5px;">Escoge
|
<label class="form-check-label" for="activeProbe" style="padding-left: 5px;">Escoge el sensor utilizado</label
|
||||||
el sensor utilizado</label>
|
>
|
||||||
<select name="activeProbe" id="activeProbe">
|
<select name="activeProbe" id="activeProbe">
|
||||||
{% for j in i.active %}
|
{% for j in i.active %}
|
||||||
<option value="{{ j }}">{{ j }}</option>
|
<option value="{{ j }}">{{ j }}</option>
|
||||||
@ -82,16 +25,23 @@
|
|||||||
|
|
||||||
<!-- Force station -->
|
<!-- Force station -->
|
||||||
<div class="form-group">
|
<div class="form-group">
|
||||||
<input class="form-check-input" type="checkbox" name="forceStation{{i.sensor}}"
|
<input
|
||||||
|
class="form-check-input"
|
||||||
|
type="checkbox"
|
||||||
|
name="forceStation{{i.sensor}}"
|
||||||
id="forceStation{{i.sensor}}"
|
id="forceStation{{i.sensor}}"
|
||||||
onchange="toggleSelect('forceStation{{i.sensor}}', 'stations{{i.sensor}}')">
|
onchange="toggleSelect('forceStation{{i.sensor}}', 'stations{{i.sensor}}')"
|
||||||
<label class="form-check-label" for="forceStation{{i.sensor}}"
|
/>
|
||||||
style="padding-left: 5px;">
|
<label
|
||||||
|
class="form-check-label"
|
||||||
|
for="forceStation{{i.sensor}}"
|
||||||
|
style="padding-left: 5px;"
|
||||||
|
>
|
||||||
Forzar estación
|
Forzar estación
|
||||||
</label>
|
</label>
|
||||||
|
|
||||||
<label class="form-check-label" for="stations{{i.sensor}}"
|
<label class="form-check-label" for="stations{{i.sensor}}" style="padding-left: 5px;">Selecciona la estación utilizada</label
|
||||||
style="padding-left: 5px;">Selecciona la estación utilizada</label>
|
>
|
||||||
<select name="stations{{i.sensor}}" id="stations{{i.sensor}}" disabled>
|
<select name="stations{{i.sensor}}" id="stations{{i.sensor}}" disabled>
|
||||||
{% for j in i.stations %}
|
{% for j in i.stations %}
|
||||||
<option value="{{ j }}">{{ j }}</option>
|
<option value="{{ j }}">{{ j }}</option>
|
||||||
@ -101,27 +51,28 @@
|
|||||||
|
|
||||||
<!-- Apply quality control -->
|
<!-- Apply quality control -->
|
||||||
<div class="list-group-item" style="padding-bottom: 50px;">
|
<div class="list-group-item" style="padding-bottom: 50px;">
|
||||||
<input class="form-check-input" type="checkbox" id="autoSizingCheck" checked>
|
<input
|
||||||
<label class="form-check-label" for="autoSizingCheck" style="padding-left: 5px;">
|
class="form-check-input"
|
||||||
|
type="checkbox"
|
||||||
|
id="autoSizingCheck"
|
||||||
|
checked
|
||||||
|
/>
|
||||||
|
<label
|
||||||
|
class="form-check-label"
|
||||||
|
for="autoSizingCheck"
|
||||||
|
style="padding-left: 5px;"
|
||||||
|
>
|
||||||
Aplicar control de calidad
|
Aplicar control de calidad
|
||||||
</label>
|
</label>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<!-- Submit -->
|
<!-- Submit -->
|
||||||
<input type="hidden" value="{{i.sensor}}" name="probe" />
|
<input type="hidden" value="{{i.sensor}}" name="probe" />
|
||||||
<button type="submit" class="btn btn-primary btn-lg btn-block">Subir CSV</button>
|
<button type="submit" class="btn btn-primary btn-lg btn-block">
|
||||||
|
Subir CSV
|
||||||
|
</button>
|
||||||
</form>
|
</form>
|
||||||
|
</div>
|
||||||
</div>
|
<!-- Close container field -->
|
||||||
</div> <!-- Close Modal Container Body -->
|
|
||||||
</div> <!-- Close Modal Body -->
|
|
||||||
</div> <!-- Close Modal Content -->
|
|
||||||
</div> <!-- Close Modal dialog -->
|
|
||||||
</div> <!-- Close Modal -->
|
|
||||||
{% endfor %}
|
|
||||||
<!-- End Sensors Modals -->
|
|
||||||
|
|
||||||
</div> <!-- Close container field -->
|
|
||||||
|
|
||||||
{% endblock %}
|
{% endblock %}
|
Loading…
Reference in New Issue
Block a user