294 lines
11 KiB
Python
294 lines
11 KiB
Python
from datetime import datetime
|
|
from shutil import copy2
|
|
import pandas as pd
|
|
import numpy as np
|
|
import os, csv
|
|
|
|
UPLOAD_FOLDER = './dataset/'
|
|
tableClass = 'table table-striped table-sm table-responsive'
|
|
|
|
def get_date(probe, file):
|
|
''' Test utils file '''
|
|
return datetime.now().strftime("%Y-%m-%d-%H:%M:%S")
|
|
|
|
# TODO:
|
|
# - Check if is a good file
|
|
#
|
|
#def checkUploadedFile(probe, file):
|
|
# return 1
|
|
|
|
# TODO:
|
|
# - clean the head of csv, need to extract info an remove the %% (maybe new file with metadata info?) [done]
|
|
# - create the empty csv, first row with headers [done]
|
|
# - desviation and more data things
|
|
def process_castaway(file):
|
|
|
|
try:
|
|
# Copy raw file to process folder
|
|
with open(file) as CastAwayFile:
|
|
filename = os.path.basename(CastAwayFile.name).strip('raw-')
|
|
processFilePath = os.path.join(
|
|
UPLOAD_FOLDER, 'CastAway', filename)
|
|
#copy2(file, processFilePath)
|
|
|
|
# Getting some metadata info
|
|
#with open(file, newline='') as castfile:
|
|
# lines = castfile.readlines()
|
|
# device = lines[0].split(',')[1].replace('\r\n', '')
|
|
# filename = lines[1].split(',')[1].replace('\r\n', '')
|
|
# start_latitude = lines[9].split(',')[1].replace('\r\n', '')
|
|
# start_longitude = lines[10].split(',')[1].replace('\r\n', '')
|
|
# start_altitude = lines[11].split(',')[1].replace('\r\n', '')
|
|
# castfile.close()
|
|
|
|
# Opening the csv with pandas
|
|
df = pd.read_csv(file, skiprows=28)
|
|
|
|
# Extract perfil bajada, el 1 es por el header
|
|
index_max = df['Depth (Meter)'].idxmax() + 1
|
|
#df_perfilBajada = df[df['depth'].between(0, df['depth'].max())] # No funciona muy bien
|
|
# Limitamos a solo el perfil de bajada
|
|
df_perfilBajada = df[:index_max]
|
|
|
|
# Guardamos solo el perfilBajada en carpeta PB
|
|
if not os.path.exists(os.path.join(UPLOAD_FOLDER, 'CastAway', 'PB')):
|
|
os.makedirs(os.path.join(UPLOAD_FOLDER, 'CastAway', 'PB'))
|
|
|
|
filenamePB = filename.strip('.csv') + '-PB.csv'
|
|
perfilBajadaFilePath = os.path.join(
|
|
UPLOAD_FOLDER, 'CastAway', 'PB', filenamePB)
|
|
df_perfilBajada.to_csv(perfilBajadaFilePath, index=False)
|
|
|
|
# Trying to show Dataframe on webpage
|
|
return df.to_html(classes=tableClass)
|
|
|
|
except Exception as ex:
|
|
print('Exception: '+repr(ex))
|
|
return ex
|
|
|
|
def process_suna(file):
|
|
try:
|
|
df = pd.read_csv(file, encoding="ISO-8859-1", header=None)
|
|
|
|
#df.columns = ['fechaHora', 'INSTRUMENT', 'Start-time', 'Nitrato(uMol/L)','Nitrato(MG/L)', 'ERROR', 'T_lamp', ]
|
|
|
|
#if not os.path.exists(os.path.join(UPLOAD_FOLDER, 'SUNA', 'HEAD')):
|
|
# os.makedirs(os.path.join(UPLOAD_FOLDER, 'SUNA', 'HEAD'))
|
|
|
|
# Return webpage
|
|
return df.to_html(classes=tableClass)
|
|
except Exception as ex:
|
|
print('Exception: '+repr(ex))
|
|
return ex
|
|
|
|
# TODO:
|
|
# - Ask for FIRe examples, actually only bin files found (done)
|
|
# - Extract file to process folder (done)
|
|
# - Save a File with PAR info
|
|
# - Headers on email (ask for new headers)[done]
|
|
# - Remove negative values (this, done)
|
|
# - Arrange similar windows depth and do measure
|
|
def process_fire(file):
|
|
''' Processing FIRe '''
|
|
|
|
try:
|
|
# Copy raw file to process folder
|
|
with open(file) as FIReFile:
|
|
filename_raw = os.path.basename(FIReFile.name)
|
|
filename = os.path.basename(FIReFile.name).strip('raw-')
|
|
#raw_file_path = os.path.join(UPLOAD_FOLDER, 'FIRe', 'raw', filename_raw)
|
|
processFilePath = os.path.join(
|
|
UPLOAD_FOLDER, 'FIRe', filename)
|
|
#copy2(file, processFilePath)
|
|
|
|
# First, need to check the csv headers
|
|
# Open the process file with pandas
|
|
df = pd.read_csv(file, header=None)
|
|
|
|
# Headers (now working fine)
|
|
df.columns = ['fechaHora', 'estacion', 'fecha', 'hora', 'profundidad', 'Fo', 'Fm', 'Fv', 'Fv/Fm', 'p', 'Abs_rel', 'Abs_abs', 'led_light', 'ETR', 'coma1', 'coma2', 'coma3', 'coma4', 'coma5', 'coma6','error_norm', 'PAR', 'V', 'cero1', 'cero2', 'cero3', 'cero4', 'raro']
|
|
df.loc[:, 'coma1'] = 0
|
|
df.loc[:, 'coma2'] = 0
|
|
df.loc[:, 'coma3'] = 0
|
|
df.loc[:, 'coma4'] = 0
|
|
df.loc[:, 'coma5'] = 0
|
|
df.loc[:, 'coma6'] = 0
|
|
|
|
# Fixing empty values
|
|
#df.to_csv(os.path.join(UPLOAD_FOLDER, 'FIRe', 'raw', 'raw-'+filename), index=False)
|
|
|
|
filename_nn = execution_NN(filename_raw)
|
|
|
|
execution_PAR(filename_nn)
|
|
|
|
return df.to_html(classes=tableClass)
|
|
|
|
except Exception as ex:
|
|
print('Exception: '+repr(ex))
|
|
return ex
|
|
|
|
# TODO:
|
|
# - Only perfil bajada (done)
|
|
def process_phyco(file):
|
|
''' Processing PhycoCTD'''
|
|
|
|
try:
|
|
# Copy raw file to process folder
|
|
with open(file, "r+") as phycoFile:
|
|
# Sustract raw- of filename
|
|
filename_raw = os.path.basename(phycoFile.name)
|
|
filename = os.path.basename(phycoFile.name).strip('raw-')
|
|
processFilePath = os.path.join(
|
|
UPLOAD_FOLDER, 'PhycoCTD', filename)
|
|
#copy2(file, processFilePath)
|
|
|
|
# Working with the process file
|
|
df = pd.read_csv(file, delimiter=';')
|
|
|
|
check_if_old_phyco(filename_raw)
|
|
execution_pb_phyco(filename_raw)
|
|
|
|
# Return webpage
|
|
return df.to_html(classes=tableClass)
|
|
except Exception as ex:
|
|
print('Exception: '+repr(ex))
|
|
return ex
|
|
|
|
def check_if_old_phyco(filename):
|
|
# Checking if old file csv
|
|
PATH = os.path.join(UPLOAD_FOLDER, 'PhycoCTD', 'raw', filename)
|
|
with open(PATH, "r+") as phycoFile:
|
|
# Check if old version
|
|
line = phycoFile.readline()
|
|
if (',' in line):
|
|
old = True
|
|
else:
|
|
old = False
|
|
|
|
if (old == True):
|
|
df_old = pd.read_csv(PATH, header=None, skiprows=1)
|
|
if (len(df_old.columns) == 16):
|
|
df_old.columns = ['station', 'latitude', 'longitude', 'time', 'depth', 'temp1', 'temp2', 'cdom[gain]',
|
|
'cdom[ppb]', 'cdom[mv]', 'pe[gain]', 'pe[ppb]', 'pe[mv]', 'chl[gain]', 'chl[ppb]', 'chl[mv]']
|
|
else:
|
|
df_old.columns = ['station', 'time', 'depth', 'temp1', 'temp2', 'cdom[gain]', 'cdom[ppb]',
|
|
'cdom[mv]', 'pe[gain]', 'pe[ppb]', 'pe[mv]', 'chl[gain]', 'chl[ppb]', 'chl[mv]']
|
|
raw_path = os.path.join(UPLOAD_FOLDER, 'PhycoCTD', 'raw', filename)
|
|
df_old.to_csv(raw_path, index=False, sep=';')
|
|
|
|
def execution_pb_phyco(filename):
|
|
# Guardamos solo el perfilBajada
|
|
if not os.path.exists(os.path.join(UPLOAD_FOLDER, 'PhycoCTD', 'PB')):
|
|
os.makedirs(os.path.join(UPLOAD_FOLDER, 'PhycoCTD', 'PB'))
|
|
|
|
PATH = os.path.join(UPLOAD_FOLDER, 'PhycoCTD', 'raw', filename)
|
|
df = pd.read_csv(PATH, delimiter=';')
|
|
|
|
# Checking if NaN values exists [for hack lab version csv upload]
|
|
if (df['temp2'].isnull().sum() > 0):
|
|
df.loc[:, 'temp2'] = 0
|
|
df = df.dropna()
|
|
|
|
# Extract perfil bajada, el 1 es por el header
|
|
index_max = df['depth'].idxmax() + 1
|
|
#df_perfilBajada = df[df['depth'].between(0, df['depth'].max())] # No funciona muy bien
|
|
# Limitamos a solo el perfil de bajada
|
|
df_pb= df[:index_max]
|
|
|
|
filename_pb = filename.strip('raw-').strip('.csv') + '-PB.csv'
|
|
pb_path = os.path.join(UPLOAD_FOLDER, 'PhycoCTD', 'PB', filename_pb)
|
|
df_pb.to_csv(pb_path, sep=';', index=False)
|
|
|
|
def execution_NN(filename):
|
|
|
|
PATH = os.path.join(UPLOAD_FOLDER, 'FIRe', 'raw', filename)
|
|
df = pd.read_csv(PATH, header=None)
|
|
|
|
# Headers (now working fine)
|
|
df.columns = ['fechaHora', 'estacion', 'fecha', 'hora', 'profundidad', 'Fo', 'Fm', 'Fv', 'Fv/Fm', 'p', 'Abs_rel', 'Abs_abs', 'led_light',
|
|
'ETR', 'coma1', 'coma2', 'coma3', 'coma4', 'coma5', 'coma6', 'error_norm', 'PAR', 'V', 'cero1', 'cero2', 'cero3', 'cero4', 'raro']
|
|
df.loc[:, 'coma1'] = 0
|
|
df.loc[:, 'coma2'] = 0
|
|
df.loc[:, 'coma3'] = 0
|
|
df.loc[:, 'coma4'] = 0
|
|
df.loc[:, 'coma5'] = 0
|
|
df.loc[:, 'coma6'] = 0
|
|
|
|
# NonNegative values rutine
|
|
if not os.path.exists(os.path.join(UPLOAD_FOLDER, 'FIRe', 'NN')):
|
|
os.makedirs(os.path.join(UPLOAD_FOLDER, 'FIRe', 'NN'))
|
|
|
|
# Remove rows with negatives values
|
|
df_nn = df[(df.iloc[:, 4:27] >= 0).all(1)]
|
|
filename_nn = filename.strip('raw-').strip('.csv') + '-NN.csv'
|
|
nonnegative_path = os.path.join(UPLOAD_FOLDER, 'FIRe', 'NN', filename_nn)
|
|
df_nn.to_csv(nonnegative_path, index=False)
|
|
|
|
return filename_nn
|
|
|
|
def execution_PAR(filename):
|
|
# PAR rutine
|
|
if not os.path.exists(os.path.join(UPLOAD_FOLDER, 'FIRe', 'PAR')):
|
|
os.makedirs(os.path.join(UPLOAD_FOLDER, 'FIRe', 'PAR'))
|
|
|
|
PATH = os.path.join(UPLOAD_FOLDER, 'FIRe', 'NN', filename)
|
|
df = pd.read_csv(PATH)
|
|
|
|
# PAR execution
|
|
PAR_columns = ['estacion', 'fecha', 'profundidad', 'Fo', 'Fm', 'Fv', 'Fv/Fm', 'p', 'Abs_rel', 'Abs_abs', 'led_light',
|
|
'ETR', 'error_norm', 'PAR']
|
|
|
|
df_PAR = pd.DataFrame(columns=PAR_columns)
|
|
|
|
index_max = df['profundidad'].idxmax()
|
|
index_min = df['profundidad'].idxmin() + 1
|
|
depth_list = df[index_max:index_min]['profundidad'].to_list()
|
|
|
|
last_value = df['profundidad'].max()
|
|
similar_depth = []
|
|
for i in range(len(depth_list)):
|
|
#print(depth_list[i])
|
|
|
|
if (abs(last_value - depth_list[i]) <= 2000):
|
|
last_value = depth_list[i]
|
|
similar_depth.append(depth_list[i])
|
|
elif (abs(last_value - depth_list[i]) >= 6000):
|
|
## Save the actual list to DataFrame
|
|
#print(similar_depth)
|
|
index_first = pd.to_numeric(df.index[df['profundidad'] == similar_depth[0]])[0]
|
|
index_last = pd.to_numeric(df.index[df['profundidad'] == similar_depth[-1]])[0] + 1
|
|
df_range = df.iloc[index_first:index_last]
|
|
|
|
# Working with df_range
|
|
estacion = df['estacion'][0]
|
|
fecha = df['fecha'][0]
|
|
profundidad = df_range['profundidad'].mean()
|
|
Fo = df_range['Fo'].mean()
|
|
Fm = df_range['Fm'].mean()
|
|
Fv = df_range['Fv'].mean()
|
|
FvFm = df_range['Fv/Fm'].mean()
|
|
p = df_range['p'].mean()
|
|
Abs_rel = df_range['Abs_rel'].mean()
|
|
Abs_abs = df_range['Abs_abs'].mean()
|
|
led_light = df_range['led_light'].mean()
|
|
ETR = df_range['ETR'].mean()
|
|
error_norm = df_range['error_norm'].mean()
|
|
PAR = df_range['PAR'].mean()
|
|
data = [estacion, fecha, profundidad, Fo, Fm, Fv, FvFm, p, Abs_rel, Abs_abs, led_light, ETR, error_norm, PAR]
|
|
row = pd.Series(data, index=PAR_columns)
|
|
df_PAR = df_PAR.append(row, ignore_index=True)
|
|
|
|
## Empty the list
|
|
similar_depth = []
|
|
last_value = depth_list[i]
|
|
|
|
## Adding to the new list
|
|
similar_depth.append(depth_list[i])
|
|
|
|
# Saving the DataFrame
|
|
# TODO: check last values, around 400 depth
|
|
filenamePAR = filename.strip('-NN.csv') + '-PAR.csv'
|
|
PARFilePath = os.path.join(UPLOAD_FOLDER, 'FIRe', 'PAR', filenamePAR)
|
|
df_PAR.to_csv(PARFilePath, index=False)
|
|
|