first commit
This commit is contained in:
293
probeutils/utils.py
Normal file
293
probeutils/utils.py
Normal file
@ -0,0 +1,293 @@
|
||||
from datetime import datetime
|
||||
from shutil import copy2
|
||||
import pandas as pd
|
||||
import numpy as np
|
||||
import os, csv
|
||||
|
||||
UPLOAD_FOLDER = './dataset/'
|
||||
tableClass = 'table table-striped table-sm table-responsive'
|
||||
|
||||
def get_date(probe, file):
|
||||
''' Test utils file '''
|
||||
return datetime.now().strftime("%Y-%m-%d-%H:%M:%S")
|
||||
|
||||
# TODO:
|
||||
# - Check if is a good file
|
||||
#
|
||||
#def checkUploadedFile(probe, file):
|
||||
# return 1
|
||||
|
||||
# TODO:
|
||||
# - clean the head of csv, need to extract info an remove the %% (maybe new file with metadata info?) [done]
|
||||
# - create the empty csv, first row with headers [done]
|
||||
# - desviation and more data things
|
||||
def process_castaway(file):
|
||||
|
||||
try:
|
||||
# Copy raw file to process folder
|
||||
with open(file) as CastAwayFile:
|
||||
filename = os.path.basename(CastAwayFile.name).strip('raw-')
|
||||
processFilePath = os.path.join(
|
||||
UPLOAD_FOLDER, 'CastAway', filename)
|
||||
#copy2(file, processFilePath)
|
||||
|
||||
# Getting some metadata info
|
||||
#with open(file, newline='') as castfile:
|
||||
# lines = castfile.readlines()
|
||||
# device = lines[0].split(',')[1].replace('\r\n', '')
|
||||
# filename = lines[1].split(',')[1].replace('\r\n', '')
|
||||
# start_latitude = lines[9].split(',')[1].replace('\r\n', '')
|
||||
# start_longitude = lines[10].split(',')[1].replace('\r\n', '')
|
||||
# start_altitude = lines[11].split(',')[1].replace('\r\n', '')
|
||||
# castfile.close()
|
||||
|
||||
# Opening the csv with pandas
|
||||
df = pd.read_csv(file, skiprows=28)
|
||||
|
||||
# Extract perfil bajada, el 1 es por el header
|
||||
index_max = df['Depth (Meter)'].idxmax() + 1
|
||||
#df_perfilBajada = df[df['depth'].between(0, df['depth'].max())] # No funciona muy bien
|
||||
# Limitamos a solo el perfil de bajada
|
||||
df_perfilBajada = df[:index_max]
|
||||
|
||||
# Guardamos solo el perfilBajada en carpeta PB
|
||||
if not os.path.exists(os.path.join(UPLOAD_FOLDER, 'CastAway', 'PB')):
|
||||
os.makedirs(os.path.join(UPLOAD_FOLDER, 'CastAway', 'PB'))
|
||||
|
||||
filenamePB = filename.strip('.csv') + '-PB.csv'
|
||||
perfilBajadaFilePath = os.path.join(
|
||||
UPLOAD_FOLDER, 'CastAway', 'PB', filenamePB)
|
||||
df_perfilBajada.to_csv(perfilBajadaFilePath, index=False)
|
||||
|
||||
# Trying to show Dataframe on webpage
|
||||
return df.to_html(classes=tableClass)
|
||||
|
||||
except Exception as ex:
|
||||
print('Exception: '+repr(ex))
|
||||
return ex
|
||||
|
||||
def process_suna(file):
|
||||
try:
|
||||
df = pd.read_csv(file, encoding="ISO-8859-1", header=None)
|
||||
|
||||
#df.columns = ['fechaHora', 'INSTRUMENT', 'Start-time', 'Nitrato(uMol/L)','Nitrato(MG/L)', 'ERROR', 'T_lamp', ]
|
||||
|
||||
#if not os.path.exists(os.path.join(UPLOAD_FOLDER, 'SUNA', 'HEAD')):
|
||||
# os.makedirs(os.path.join(UPLOAD_FOLDER, 'SUNA', 'HEAD'))
|
||||
|
||||
# Return webpage
|
||||
return df.to_html(classes=tableClass)
|
||||
except Exception as ex:
|
||||
print('Exception: '+repr(ex))
|
||||
return ex
|
||||
|
||||
# TODO:
|
||||
# - Ask for FIRe examples, actually only bin files found (done)
|
||||
# - Extract file to process folder (done)
|
||||
# - Save a File with PAR info
|
||||
# - Headers on email (ask for new headers)[done]
|
||||
# - Remove negative values (this, done)
|
||||
# - Arrange similar windows depth and do measure
|
||||
def process_fire(file):
|
||||
''' Processing FIRe '''
|
||||
|
||||
try:
|
||||
# Copy raw file to process folder
|
||||
with open(file) as FIReFile:
|
||||
filename_raw = os.path.basename(FIReFile.name)
|
||||
filename = os.path.basename(FIReFile.name).strip('raw-')
|
||||
#raw_file_path = os.path.join(UPLOAD_FOLDER, 'FIRe', 'raw', filename_raw)
|
||||
processFilePath = os.path.join(
|
||||
UPLOAD_FOLDER, 'FIRe', filename)
|
||||
#copy2(file, processFilePath)
|
||||
|
||||
# First, need to check the csv headers
|
||||
# Open the process file with pandas
|
||||
df = pd.read_csv(file, header=None)
|
||||
|
||||
# Headers (now working fine)
|
||||
df.columns = ['fechaHora', 'estacion', 'fecha', 'hora', 'profundidad', 'Fo', 'Fm', 'Fv', 'Fv/Fm', 'p', 'Abs_rel', 'Abs_abs', 'led_light', 'ETR', 'coma1', 'coma2', 'coma3', 'coma4', 'coma5', 'coma6','error_norm', 'PAR', 'V', 'cero1', 'cero2', 'cero3', 'cero4', 'raro']
|
||||
df.loc[:, 'coma1'] = 0
|
||||
df.loc[:, 'coma2'] = 0
|
||||
df.loc[:, 'coma3'] = 0
|
||||
df.loc[:, 'coma4'] = 0
|
||||
df.loc[:, 'coma5'] = 0
|
||||
df.loc[:, 'coma6'] = 0
|
||||
|
||||
# Fixing empty values
|
||||
#df.to_csv(os.path.join(UPLOAD_FOLDER, 'FIRe', 'raw', 'raw-'+filename), index=False)
|
||||
|
||||
filename_nn = execution_NN(filename_raw)
|
||||
|
||||
execution_PAR(filename_nn)
|
||||
|
||||
return df.to_html(classes=tableClass)
|
||||
|
||||
except Exception as ex:
|
||||
print('Exception: '+repr(ex))
|
||||
return ex
|
||||
|
||||
# TODO:
|
||||
# - Only perfil bajada (done)
|
||||
def process_phyco(file):
|
||||
''' Processing PhycoCTD'''
|
||||
|
||||
try:
|
||||
# Copy raw file to process folder
|
||||
with open(file, "r+") as phycoFile:
|
||||
# Sustract raw- of filename
|
||||
filename_raw = os.path.basename(phycoFile.name)
|
||||
filename = os.path.basename(phycoFile.name).strip('raw-')
|
||||
processFilePath = os.path.join(
|
||||
UPLOAD_FOLDER, 'PhycoCTD', filename)
|
||||
#copy2(file, processFilePath)
|
||||
|
||||
# Working with the process file
|
||||
df = pd.read_csv(file, delimiter=';')
|
||||
|
||||
check_if_old_phyco(filename_raw)
|
||||
execution_pb_phyco(filename_raw)
|
||||
|
||||
# Return webpage
|
||||
return df.to_html(classes=tableClass)
|
||||
except Exception as ex:
|
||||
print('Exception: '+repr(ex))
|
||||
return ex
|
||||
|
||||
def check_if_old_phyco(filename):
|
||||
# Checking if old file csv
|
||||
PATH = os.path.join(UPLOAD_FOLDER, 'PhycoCTD', 'raw', filename)
|
||||
with open(PATH, "r+") as phycoFile:
|
||||
# Check if old version
|
||||
line = phycoFile.readline()
|
||||
if (',' in line):
|
||||
old = True
|
||||
else:
|
||||
old = False
|
||||
|
||||
if (old == True):
|
||||
df_old = pd.read_csv(PATH, header=None, skiprows=1)
|
||||
if (len(df_old.columns) == 16):
|
||||
df_old.columns = ['station', 'latitude', 'longitude', 'time', 'depth', 'temp1', 'temp2', 'cdom[gain]',
|
||||
'cdom[ppb]', 'cdom[mv]', 'pe[gain]', 'pe[ppb]', 'pe[mv]', 'chl[gain]', 'chl[ppb]', 'chl[mv]']
|
||||
else:
|
||||
df_old.columns = ['station', 'time', 'depth', 'temp1', 'temp2', 'cdom[gain]', 'cdom[ppb]',
|
||||
'cdom[mv]', 'pe[gain]', 'pe[ppb]', 'pe[mv]', 'chl[gain]', 'chl[ppb]', 'chl[mv]']
|
||||
raw_path = os.path.join(UPLOAD_FOLDER, 'PhycoCTD', 'raw', filename)
|
||||
df_old.to_csv(raw_path, index=False, sep=';')
|
||||
|
||||
def execution_pb_phyco(filename):
|
||||
# Guardamos solo el perfilBajada
|
||||
if not os.path.exists(os.path.join(UPLOAD_FOLDER, 'PhycoCTD', 'PB')):
|
||||
os.makedirs(os.path.join(UPLOAD_FOLDER, 'PhycoCTD', 'PB'))
|
||||
|
||||
PATH = os.path.join(UPLOAD_FOLDER, 'PhycoCTD', 'raw', filename)
|
||||
df = pd.read_csv(PATH, delimiter=';')
|
||||
|
||||
# Checking if NaN values exists [for hack lab version csv upload]
|
||||
if (df['temp2'].isnull().sum() > 0):
|
||||
df.loc[:, 'temp2'] = 0
|
||||
df = df.dropna()
|
||||
|
||||
# Extract perfil bajada, el 1 es por el header
|
||||
index_max = df['depth'].idxmax() + 1
|
||||
#df_perfilBajada = df[df['depth'].between(0, df['depth'].max())] # No funciona muy bien
|
||||
# Limitamos a solo el perfil de bajada
|
||||
df_pb= df[:index_max]
|
||||
|
||||
filename_pb = filename.strip('raw-').strip('.csv') + '-PB.csv'
|
||||
pb_path = os.path.join(UPLOAD_FOLDER, 'PhycoCTD', 'PB', filename_pb)
|
||||
df_pb.to_csv(pb_path, sep=';', index=False)
|
||||
|
||||
def execution_NN(filename):
|
||||
|
||||
PATH = os.path.join(UPLOAD_FOLDER, 'FIRe', 'raw', filename)
|
||||
df = pd.read_csv(PATH, header=None)
|
||||
|
||||
# Headers (now working fine)
|
||||
df.columns = ['fechaHora', 'estacion', 'fecha', 'hora', 'profundidad', 'Fo', 'Fm', 'Fv', 'Fv/Fm', 'p', 'Abs_rel', 'Abs_abs', 'led_light',
|
||||
'ETR', 'coma1', 'coma2', 'coma3', 'coma4', 'coma5', 'coma6', 'error_norm', 'PAR', 'V', 'cero1', 'cero2', 'cero3', 'cero4', 'raro']
|
||||
df.loc[:, 'coma1'] = 0
|
||||
df.loc[:, 'coma2'] = 0
|
||||
df.loc[:, 'coma3'] = 0
|
||||
df.loc[:, 'coma4'] = 0
|
||||
df.loc[:, 'coma5'] = 0
|
||||
df.loc[:, 'coma6'] = 0
|
||||
|
||||
# NonNegative values rutine
|
||||
if not os.path.exists(os.path.join(UPLOAD_FOLDER, 'FIRe', 'NN')):
|
||||
os.makedirs(os.path.join(UPLOAD_FOLDER, 'FIRe', 'NN'))
|
||||
|
||||
# Remove rows with negatives values
|
||||
df_nn = df[(df.iloc[:, 4:27] >= 0).all(1)]
|
||||
filename_nn = filename.strip('raw-').strip('.csv') + '-NN.csv'
|
||||
nonnegative_path = os.path.join(UPLOAD_FOLDER, 'FIRe', 'NN', filename_nn)
|
||||
df_nn.to_csv(nonnegative_path, index=False)
|
||||
|
||||
return filename_nn
|
||||
|
||||
def execution_PAR(filename):
|
||||
# PAR rutine
|
||||
if not os.path.exists(os.path.join(UPLOAD_FOLDER, 'FIRe', 'PAR')):
|
||||
os.makedirs(os.path.join(UPLOAD_FOLDER, 'FIRe', 'PAR'))
|
||||
|
||||
PATH = os.path.join(UPLOAD_FOLDER, 'FIRe', 'NN', filename)
|
||||
df = pd.read_csv(PATH)
|
||||
|
||||
# PAR execution
|
||||
PAR_columns = ['estacion', 'fecha', 'profundidad', 'Fo', 'Fm', 'Fv', 'Fv/Fm', 'p', 'Abs_rel', 'Abs_abs', 'led_light',
|
||||
'ETR', 'error_norm', 'PAR']
|
||||
|
||||
df_PAR = pd.DataFrame(columns=PAR_columns)
|
||||
|
||||
index_max = df['profundidad'].idxmax()
|
||||
index_min = df['profundidad'].idxmin() + 1
|
||||
depth_list = df[index_max:index_min]['profundidad'].to_list()
|
||||
|
||||
last_value = df['profundidad'].max()
|
||||
similar_depth = []
|
||||
for i in range(len(depth_list)):
|
||||
#print(depth_list[i])
|
||||
|
||||
if (abs(last_value - depth_list[i]) <= 2000):
|
||||
last_value = depth_list[i]
|
||||
similar_depth.append(depth_list[i])
|
||||
elif (abs(last_value - depth_list[i]) >= 6000):
|
||||
## Save the actual list to DataFrame
|
||||
#print(similar_depth)
|
||||
index_first = pd.to_numeric(df.index[df['profundidad'] == similar_depth[0]])[0]
|
||||
index_last = pd.to_numeric(df.index[df['profundidad'] == similar_depth[-1]])[0] + 1
|
||||
df_range = df.iloc[index_first:index_last]
|
||||
|
||||
# Working with df_range
|
||||
estacion = df['estacion'][0]
|
||||
fecha = df['fecha'][0]
|
||||
profundidad = df_range['profundidad'].mean()
|
||||
Fo = df_range['Fo'].mean()
|
||||
Fm = df_range['Fm'].mean()
|
||||
Fv = df_range['Fv'].mean()
|
||||
FvFm = df_range['Fv/Fm'].mean()
|
||||
p = df_range['p'].mean()
|
||||
Abs_rel = df_range['Abs_rel'].mean()
|
||||
Abs_abs = df_range['Abs_abs'].mean()
|
||||
led_light = df_range['led_light'].mean()
|
||||
ETR = df_range['ETR'].mean()
|
||||
error_norm = df_range['error_norm'].mean()
|
||||
PAR = df_range['PAR'].mean()
|
||||
data = [estacion, fecha, profundidad, Fo, Fm, Fv, FvFm, p, Abs_rel, Abs_abs, led_light, ETR, error_norm, PAR]
|
||||
row = pd.Series(data, index=PAR_columns)
|
||||
df_PAR = df_PAR.append(row, ignore_index=True)
|
||||
|
||||
## Empty the list
|
||||
similar_depth = []
|
||||
last_value = depth_list[i]
|
||||
|
||||
## Adding to the new list
|
||||
similar_depth.append(depth_list[i])
|
||||
|
||||
# Saving the DataFrame
|
||||
# TODO: check last values, around 400 depth
|
||||
filenamePAR = filename.strip('-NN.csv') + '-PAR.csv'
|
||||
PARFilePath = os.path.join(UPLOAD_FOLDER, 'FIRe', 'PAR', filenamePAR)
|
||||
df_PAR.to_csv(PARFilePath, index=False)
|
||||
|
||||
Reference in New Issue
Block a user