code: Parsing HUE data set

code/parse_hue.py

@@ -0,0 +1,101 @@
+#!/usr/bin/env python3
+
+import sys
+sys.path.insert(1, 'lib')
+import argparse
+import ast
+import csv
+from datetime import datetime
+from geopy.distance import distance
+import io
+import itertools
+import lmdk_lib
+import numpy as np
+import os
+import matplotlib.pyplot as plt
+import time
+import zipfile
+
+
+# https://cloud.delkappa.com/s/oNWLmM6jrpjK8Ff
+# HUE data format
+# Header size
+hdr = 1
+# Date
+dt = 0
+# Hour
+hr = 1
+# Energy consumption in kWh
+kwh = 2
+# Timestamp format
+tim_fmt = "%Y-%m-%d %H"
+
+
+def main(args):
+  '''
+    Load data
+  '''
+  # Get consumption data from previous parsing
+  cons_data = lmdk_lib.load_data(args, 'cons')
+  if cons_data.size == 0:
+    # Consumption [dt, hr, kwh]
+    cons = []
+    try:
+      print('Extracting %s... ' %(os.path.abspath(args.arc)), end='', flush=True)
+      with zipfile.ZipFile(args.arc, 'r') as arc:
+        print('[OK]')
+        with io.TextIOWrapper(arc.open(args.dat), newline='\n') as dat:
+          try:
+            print('Finding consumption... ', end='', flush=True)
+            # Get the consumption by skipping the header
+            cons_l = list(csv.reader(dat, delimiter=','))[hdr:]
+            # Check each line
+            for c in cons_l:
+              # Add valid consumption
+              if c[kwh] != '' and float(c[kwh]) > 0:
+                cons.append([datetime.strptime(c[dt] + ' ' + c[hr], tim_fmt).timestamp(), c[kwh]])
+            print('[OK]')
+          except Exception as e:
+            print('[Error: %s]' %(e))
+    except Exception as e:
+      print('[Error: %s]' %(e))
+    # Save to results
+    lmdk_lib.save_data(args, np.array(cons, np.float32), 'cons')
+
+
+'''
+  Parse arguments.
+
+  Optional:
+    arc - The data archive file.
+    dat - The consumption data file.
+    res - The results archive file.
+'''
+def parse_args():
+  # Create argument parser.
+  parser = argparse.ArgumentParser()
+
+  # Mandatory arguments.
+
+  # Optional arguments.
+  parser.add_argument('-a', '--arc', help='The data archive file.', type=str, default='/home/manos/Cloud/Data/HUE/Data.zip')
+  parser.add_argument('-d', '--dat', help='The consumption data file.', type=str, default='Residential_26.csv')
+  parser.add_argument('-r', '--res', help='The results archive file.', type=str, default='/home/manos/Cloud/Data/HUE/Results.zip')
+
+  # Parse arguments.
+  args = parser.parse_args()
+
+  return args
+
+
+if __name__ == '__main__':
+  try:
+    start_time = time.time()
+    main(parse_args())
+    end_time = time.time()
+    print('##############################')
+    print('Time   : %.4fs' % (end_time - start_time))
+    print('##############################')
+  except KeyboardInterrupt:
+    print('Interrupted by user.')
+    exit()