Merge branch 'master'

Handle missing hourly price data from ENTSOE API by interpolating
values from adjacent time slots instead of returning nil.

.dockerignore

@@ -0,0 +1,2 @@
+.git
+data

.gitignore

@@ -1,177 +1,5 @@
-# ---> Python
+.local
-# Byte-compiled / optimized / DLL files
+.config
-__pycache__/
+.project
-*.py[cod]
+*.pid
-*$py.class
-
-# C extensions
-*.so
-
-# Distribution / packaging
-.Python
-build/
-develop-eggs/
-dist/
-downloads/
-eggs/
-.eggs/
-lib/
-lib64/
-parts/
-sdist/
-var/
-wheels/
-share/python-wheels/
-*.egg-info/
-.installed.cfg
-*.egg
-MANIFEST
-
-# PyInstaller
-#  Usually these files are written by a python script from a template
-#  before PyInstaller builds the exe, so as to inject date/other infos into it.
-*.manifest
-*.spec
-
-# Installer logs
-pip-log.txt
-pip-delete-this-directory.txt
-
-# Unit test / coverage reports
-htmlcov/
-.tox/
-.nox/
-.coverage
-.coverage.*
-.cache
-nosetests.xml
-coverage.xml
-*.cover
-*.py,cover
-.hypothesis/
-.pytest_cache/
-cover/
-
-# Translations
-*.mo
-*.pot
-
-# Django stuff:
-*.log
-local_settings.py
-db.sqlite3
-db.sqlite3-journal
-
-# Flask stuff:
-instance/
-.webassets-cache
-
-# Scrapy stuff:
-.scrapy
-
-# Sphinx documentation
-docs/_build/
-
-# PyBuilder
-.pybuilder/
-target/
-
-# Jupyter Notebook
-.ipynb_checkpoints
-
-# IPython
-profile_default/
-ipython_config.py
-
-# pyenv
-#   For a library or package, you might want to ignore these files since the code is
-#   intended to run in multiple environments; otherwise, check them in:
-# .python-version
-
-# pipenv
-#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
-#   However, in case of collaboration, if having platform-specific dependencies or dependencies
-#   having no cross-platform support, pipenv may install dependencies that don't work, or not
-#   install all needed dependencies.
-#Pipfile.lock
-
-# UV
-#   Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control.
-#   This is especially recommended for binary packages to ensure reproducibility, and is more
-#   commonly ignored for libraries.
-#uv.lock
-
-# poetry
-#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
-#   This is especially recommended for binary packages to ensure reproducibility, and is more
-#   commonly ignored for libraries.
-#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
-#poetry.lock
-
-# pdm
-#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
-#pdm.lock
-#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
-#   in version control.
-#   https://pdm.fming.dev/latest/usage/project/#working-with-version-control
-.pdm.toml
-.pdm-python
-.pdm-build/
-
-# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
-__pypackages__/
-
-# Celery stuff
-celerybeat-schedule
-celerybeat.pid
-
-# SageMath parsed files
-*.sage.py
-
-# Environments
-.env
-.venv
-env/
-venv/
-ENV/
-env.bak/
-venv.bak/
-
-# Spyder project settings
-.spyderproject
-.spyproject
-
-# Rope project settings
-.ropeproject
-
-# mkdocs documentation
-/site
-
-# mypy
-.mypy_cache/
-.dmypy.json
-dmypy.json
-
-# Pyre type checker
-.pyre/
-
-# pytype static type analyzer
-.pytype/
-
-# Cython debug symbols
-cython_debug/
-
-# PyCharm
-#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
-#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
-#  and can be added to the global gitignore or merged into this file.  For a more nuclear
-#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
-#.idea/
-
-# Ruff stuff:
-.ruff_cache/
-
-# PyPI configuration file
-.pypirc
-
 data

.project

@@ -0,0 +1,12 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<projectDescription>
+	<name>smartmeter</name>
+	<comment></comment>
+	<projects>
+	</projects>
+	<buildSpec>
+	</buildSpec>
+	<natures>
+		<nature>com.aptana.ruby.core.rubynature</nature>
+	</natures>
+</projectDescription>

.ruby-gemset

@@ -0,0 +1 @@
+smartmeter

.ruby-version

@@ -0,0 +1 @@
+ruby-2.7

Dockerfile

@@ -0,0 +1,21 @@
+FROM ruby:2.7
+
+ENV BUILD_PACKAGES="apt-utils build-essential curl less nodejs sudo wget zsh libmariadb-dev libserialport-dev cron"
+
+# throw errors if Gemfile has been modified since Gemfile.lock
+RUN \
+  bundle config --global frozen 1
+ 
+
+WORKDIR /usr/src/app
+
+COPY Gemfile Gemfile.lock ./
+
+RUN \
+  apt-get update -qq && \
+  apt-get install -y $BUILD_PACKAGES && \
+  bundle install
+
+COPY . .
+
+CMD ["/bin/bash -c ruby ./smartmeter.rb"]

Gemfile

@@ -0,0 +1,14 @@
+source "https://rubygems.org"
+
+gem "activerecord"
+gem "mysql2"
+gem "serialport"
+gem "state_pattern"
+gem 'rufus-scheduler'
+gem 'daemons'
+gem 'mail'
+gem 'nokogiri'
+gem 'numo-narray'
+gem 'i18n'
+gem 'gruff'
+gem 'net-http' # to avoid error: uninitialized constant ...

Gemfile.lock

@@ -0,0 +1,103 @@
+GEM
+  remote: https://rubygems.org/
+  specs:
+    activemodel (7.1.5.1)
+      activesupport (= 7.1.5.1)
+    activerecord (7.1.5.1)
+      activemodel (= 7.1.5.1)
+      activesupport (= 7.1.5.1)
+      timeout (>= 0.4.0)
+    activesupport (7.1.5.1)
+      base64
+      benchmark (>= 0.3)
+      bigdecimal
+      concurrent-ruby (~> 1.0, >= 1.0.2)
+      connection_pool (>= 2.2.5)
+      drb
+      i18n (>= 1.6, < 2)
+      logger (>= 1.4.2)
+      minitest (>= 5.1)
+      mutex_m
+      securerandom (>= 0.3)
+      tzinfo (~> 2.0)
+    base64 (0.2.0)
+    benchmark (0.4.0)
+    bigdecimal (3.1.9)
+    concurrent-ruby (1.3.4)
+    connection_pool (2.4.1)
+    daemons (1.4.1)
+    date (3.4.1)
+    drb (2.2.1)
+    et-orbi (1.2.11)
+      tzinfo
+    fugit (1.11.1)
+      et-orbi (~> 1, >= 1.2.11)
+      raabro (~> 1.4)
+    gruff (0.24.0)
+      histogram
+      rmagick (>= 5.3)
+    histogram (0.2.4.1)
+    i18n (1.14.6)
+      concurrent-ruby (~> 1.0)
+    logger (1.6.4)
+    mail (2.8.1)
+      mini_mime (>= 0.1.1)
+      net-imap
+      net-pop
+      net-smtp
+    mini_mime (1.1.5)
+    mini_portile2 (2.8.8)
+    minitest (5.25.4)
+    mutex_m (0.3.0)
+    mysql2 (0.5.6)
+    net-http (0.6.0)
+      uri
+    net-imap (0.4.18)
+      date
+      net-protocol
+    net-pop (0.1.2)
+      net-protocol
+    net-protocol (0.2.2)
+      timeout
+    net-smtp (0.5.0)
+      net-protocol
+    nokogiri (1.15.7)
+      mini_portile2 (~> 2.8.2)
+      racc (~> 1.4)
+    numo-narray (0.9.2.1)
+    observer (0.1.2)
+    pkg-config (1.5.8)
+    raabro (1.4.0)
+    racc (1.8.1)
+    rmagick (5.5.0)
+      observer (~> 0.1)
+      pkg-config (~> 1.4)
+    rufus-scheduler (3.9.2)
+      fugit (~> 1.1, >= 1.11.1)
+    securerandom (0.3.2)
+    serialport (1.3.2)
+    state_pattern (2.0.2)
+    timeout (0.4.3)
+    tzinfo (2.0.6)
+      concurrent-ruby (~> 1.0)
+    uri (1.0.2)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  activerecord
+  daemons
+  gruff
+  i18n
+  mail
+  mysql2
+  net-http
+  nokogiri
+  numo-narray
+  rufus-scheduler
+  serialport
+  state_pattern
+
+BUNDLED WITH
+   2.1.4

README.md

@@ -1,3 +1,36 @@
-# smartmeter
+ActiveRecord Without Rails
+==========================
 
-Read and store data from my smartmeter
+Just a simple example of using ActiveRecord migrations without Rails
+
+tasks you can do:
+
+* `rake db:create`
+* `rake db:migrate`
+* `rake db:drop`
+
+Or, you can run the thing to show that it'll connect
+
+```
+ruby ar-no-rails
+```
+
+Output:
+> Count of Pages: 0
+
+Lastly, you can IRB it to do stuff:
+
+$ irb
+
+```
+>> require "./ar-no-rails"
+=> true
+>> Page.new
+=> #<Page id: nil, content: nil, published: false>
+>> Page.create content: "the-content"
+=> #<Page id: 1, content: "the-content", published: false>
+```
+
+Copyright
+---------
+None. Really.

Rakefile

@@ -0,0 +1,34 @@
+require "rubygems"
+require "bundler/setup"
+
+require 'mysql2'
+require 'active_record'
+require 'yaml'
+
+namespace :db do
+
+  desc "Migrate the db"
+  task :migrate do
+    connection_details = YAML::load(File.open('config/database.yml'))
+    ActiveRecord::Base.establish_connection(connection_details)
+    ActiveRecord::MigrationContext.new("db/migrate/").migrate
+  end
+
+  desc "Create the db"
+  task :create do
+    connection_details = YAML::load(File.open('config/database.yml'))
+    admin_connection = connection_details.merge({'database'=> 'mysql', 
+                                                'schema_search_path'=> 'public'}) 
+    ActiveRecord::Base.establish_connection(admin_connection)
+    ActiveRecord::Base.connection.create_database(connection_details.fetch('database'))
+  end
+
+  desc "drop the db"
+  task :drop do
+    connection_details = YAML::load(File.open('config/database.yml'))
+    admin_connection = connection_details.merge({'database'=> 'mysql', 
+                                                'schema_search_path'=> 'public'}) 
+    ActiveRecord::Base.establish_connection(admin_connection)
+    ActiveRecord::Base.connection.drop_database(connection_details.fetch('database'))
+  end
+end

app/helpers/ConfirmingSyncLossState.rb

@@ -0,0 +1,5 @@
+class ConfirmingSyncLossState < StatePattern::State
+  def handle_byte_stream(bytes)
+    p "Please override"
+  end
+end

app/helpers/ConfirmingSyncPatternState.rb

@@ -0,0 +1,21 @@
+class ConfirmingSyncPatternState < StatePattern::State
+  # Assumes that bytes[0] == Synchronizer::SYNC_PATTERN[0]
+  def handle_byte_stream(bytes)
+   idx = 0;
+   sync_length = Synchronizer::SYNC_PATTERN.length
+   
+   # confirm rest of sync pattern 
+   while (idx < bytes.length && idx < sync_length && bytes[idx] == Synchronizer::SYNC_PATTERN[idx]) do idx = idx+1 end
+   
+   if (idx == sync_length)
+     p "Sync pattern confirmed"
+     transition_to(InSyncState)
+   else  
+     p "Back to SearchingForSync state. idx = #{idx}."
+     transition_to(SearchingForSyncState)
+   end
+  
+  # return the rest
+  return bytes[idx+1..-1] || ""
+  end
+end

app/helpers/InSyncState.rb

@@ -0,0 +1,109 @@
+require "socket"
+
+class InSyncState < StatePattern::State
+  
+  END_OF_FRAME = "!****\n"
+
+#  def initialize(stateful, previous_state)
+#      # open socket to EmonHub
+#      @hub = TCPSocket.new 'raspberrypi.lan', 5050
+#      @hub.setsockopt(Socket::SOL_SOCKET, Socket::SO_KEEPALIVE, true)
+#      super(stateful, previous_state)
+#  end
+
+
+  def handle_byte_stream(bytes)
+    idx = 0
+    #sync_pattern_length = Synchronizer::SYNC_PATTERN.length
+    sync_pattern_length = END_OF_FRAME.length
+    #p "Sync length: #{sync_length}"
+      
+    frame = ""
+
+    while (idx+sync_pattern_length < bytes.length && !new_frame_starts(bytes,idx,sync_pattern_length)) do
+      frame = frame + bytes[idx]
+      idx = idx +1     
+    end
+        
+    # did we reach the end of the frame?
+    if new_frame_starts(bytes,idx,sync_pattern_length)
+      frame_lines = frame.split("\n")
+      puts "--- FRAME  ---"
+      # p frame_lines 
+      # p "##################"
+      reading = handle_frame(frame_lines)
+      p reading 
+      return bytes[idx+sync_pattern_length..-1] || ""
+    else
+      return bytes     
+    end
+  end
+  
+  private
+     def new_frame_starts(bytes,idx,sync_pattern_length)
+       #return bytes[idx..idx+sync_pattern_length-1].eql?(Synchronizer::SYNC_PATTERN)
+       #return bytes[idx..idx+sync_pattern_length-1].eql?(END_OF_FRAME)
+       return bytes[idx].eql?(END_OF_FRAME[0])
+     end
+     
+     def handle_frame(frame_lines)
+
+       # prepare DB record
+       last_reading = Reading.last
+       reading = Reading.new
+       
+       frame_lines.each {| line|
+           if line.match(/1-0:1.8.1/) # Verbruik hoog tarief
+             reading.total_kwh_consumed_high = line.split(/1-0:1.8.1\(|\*kWh\)/).join.to_f
+             # p "Total kwh consumed (high): #{total_kwh_consumed_high}."
+           end
+           if line.match(/1-0:1.8.2/) # Verbruik laag tarief 
+             reading.total_kwh_consumed_low = line.split(/1-0:1.8.2\(|\*kWh\)/).join.to_f
+             # p "Total kwh consumed (low): #{total_kwh_consumed_low}."
+           end
+           if line.match(/1-0:2.8.1/) # Teruglevering hoog tarief
+             reading.total_kwh_produced_high = line.split(/1-0:2.8.1\(|\*kWh\)/).join.to_f
+             # p "Total kwh produced (high): #{total_kwh_produced_high}."
+           end
+           if line.match(/1-0:2.8.2/) # Teruglevering laag tarief 
+             reading.total_kwh_produced_low = line.split(/1-0:2.8.2\(|\*kWh\)/).join.to_f
+             # p "Total kwh produced (low): #{total_kwh_produced_low}."
+           end
+           if line.match(/1-0:1.7.0/) # Actueel verbruik
+             reading.current_kw_consumed = line.split(/1-0:1.7.0\(|\*kW\)/).join.to_f
+             #p "Current kW consumed: #{current_kw_consumed}."
+           end 
+           if line.match(/1-0:2.7.0/) # Actueel terug 
+             reading.current_kw_produced = line.split(/1-0:2.7.0\(|\*kW\)/).join.to_f
+           end 
+           if line.match(/0-0:96.14.0/) # Hoog/laag tarief 
+            reading.high_tarif = line.split(/0-0:96.14.0\(|\)/).join.eql?("0002")
+           end 
+           # example line: "0-1:24.2.1(250717121000S)(00000.474*m3)"
+           if match = line.match(/^(0-1:24.2.1)\(([^)]+)\)\(([\d.]+)\*m3\)$/) # Gas verbruik (1x per uur een nieuwe stand) 
+            #p "Gas reading: #{match[1]} (#{match[2]})"
+            #datetime = DateTime.strptime(match[2][0..11], "%y%m%d%H%M%S")
+            #p "Gas reading at #{datetime}."
+            reading.total_m3_gas_consumed = match[3].to_f
+           end                
+         }
+        
+        if last_reading && last_reading.eql_reading?(reading)
+          p "Nothing changed. Do not add to the database"
+        else                        
+          reading.save
+        end
+
+	# Write to EmonHub
+        begin
+		TCPSocket.open("printserver",5050){|s|
+	  s.write(sprintf("8 %d %d\r\n", reading.current_kw_consumed*1000, reading.current_kw_produced*1000))
+	 }
+	rescue
+	 p "Socket problem."
+	end
+	# Result
+        return reading
+
+     end
+end

app/helpers/ReadingsMailer.rb

@@ -0,0 +1,146 @@
+require "mail"
+
+class ReadingsMailer
+    
+  SSL_OPTS = {
+                :openssl_verify_mode => OpenSSL::SSL::VERIFY_NONE,
+  }  
+  
+  # IMAP_OPTS = { :address    => "mail.van-halteren.net",
+                                # :port       => 993,
+                                # :user_name  => 'aart@van-halteren.net',
+                                # :password   => 'XXXXX',
+                                # :openssl_verify_mode => OpenSSL::SSL::VERIFY_NONE,
+                                # :enable_ssl => true
+  # }
+                              
+  #
+  # Class methods
+  #
+  class << self
+    def deliver(date)     
+    # Read SMTP options from smtp.yml
+    smtp_opts = YAML::load(File.open('config/smtp.yml')).symbolize_keys
+    smtp_opts.merge!(SSL_OPTS) if smtp_opts[:ssl] && smtp_opts[:ssl_verify_mode].eql?("none")
+    
+    # Fetch today's usage
+    usage_today = Reading.diff_on(date)
+    consumption_today, production_today = Reading.consumed_and_produced_for_diff(usage_today)
+    net_consumption_high, net_consumption_low = Reading.net_consumed_high_and_low_for_diff(usage_today)
+
+    # Calculate costs for oxxio and easy energy
+    c = Cost.new
+    oxxio_normaal_cost, oxxio_dal_cost = c.oxxio_energy_cost(date.to_s,net_consumption_high,net_consumption_low)
+    oxxio_cost = oxxio_normaal_cost+oxxio_dal_cost
+    easy_cost = c.easy_energy_cost_barplot(date) # side effect: generates a PNG
+    # rounding
+    oxxio_cost = oxxio_cost.round(2)
+    easy_cost = easy_cost.round(2)
+    
+    mail = Mail.new do
+        delivery_method :smtp, smtp_opts
+        to      'a.t.van.halteren@vu.nl'
+        from    'SmartMeter <aart@van-halteren.net>'
+        subject "SmartMeter report for #{date}"
+
+        text_part do
+          body "Summary for #{date}\n
+                -------------------------------\n\n
+                Total kWH electricity consumed: #{consumption_today}\n
+                Total kWH electricity produced:  #{production_today}\n
+                Total m3 gas consumed: #{usage_today[:total_m3_gas_consumed]}\n\n
+                kWH cost (Oxxio): EUR #{ oxxio_cost }\n
+                kWH cost (EasyEnergy): EUR #{ easy_cost }\n
+               "
+        end
+      
+        html_part do
+          content_type 'text/html; charset=UTF-8'
+          body "<h1>Summary for #{date}</h1>" +
+               "<p>Total kWH electricity consumed: #{consumption_today}</p>" +
+               "<p>Total kWH electricity produced:  #{production_today}</p>" +
+               "<p>Total m3 gas consumed: #{usage_today[:total_m3_gas_consumed]}</p>" +
+               "</br>" +
+               "<p>kWH cost (Oxxio): <b>EUR #{ oxxio_cost} </b></p>" +
+               "<p>kWH cost (EasyEnergy): <b>EUR #{ easy_cost} </b></p>"
+        end 
+        
+        # add attachment
+        filename = "easy_cost_%s.png" % date.strftime("%F")
+        add_file :filename => filename, :content => File.read("plots/%s" % filename) 
+      end
+      
+      mail.deliver!
+    end
+
+    # default is current year
+    def deliver_for_month(month, year=nil)
+      return if (month <1 || month >12)
+
+      year = Date.today.year unless year
+      date = Date.parse("%s%.2d01" % [year,month])
+      date_str = date.strftime("%B %Y")
+      
+      # Read SMTP options from smtp.yml
+      smtp_opts = YAML::load(File.open('config/smtp.yml')).symbolize_keys
+      smtp_opts.merge!(SSL_OPTS) if smtp_opts[:ssl] && smtp_opts[:ssl_verify_mode].eql?("none")
+    
+      # Fetch today's usage
+      usage_month = Reading.diff_in_month(date)
+      consumption_month, production_month = Reading.consumed_and_produced_for_diff(usage_month)
+      net_consumption_normaal, net_consumption_dal = Reading.net_consumed_high_and_low_for_diff(usage_month)
+
+
+      # Calculate costs for oxxio and easy energy
+      c = Cost.new
+      vat = 1 + c.vat_at(date)
+
+      # cost with VAT, but without energiebelasting.
+      oxxio_raw_cost = c.oxxio_energy_cost(date.to_s,net_consumption_normaal, net_consumption_dal, false).sum.round(2)
+      easy_energy_raw_cost = c.easy_energy_raw_cost_in_month(date).compact.sum.round(2)
+
+      # cost with VAT and with energiebelasting
+      oxxio_cost = c.oxxio_energy_cost(date.to_s,net_consumption_normaal, net_consumption_dal).sum.round(2)
+      easy_energy_cost = c.easy_energy_cost_in_month(date).compact.sum.round(2)
+
+      # opslag Easy Energy
+      easy_energy_opslag = (c.easy_energy_rate(date.to_s)*vat*(net_consumption_normaal + net_consumption_dal)).round(2)
+
+      mail = Mail.new do
+        delivery_method :smtp, smtp_opts
+        to      'a.t.van.halteren@vu.nl'
+        from    'SmartMeter <aart@van-halteren.net>'
+        subject "SmartMeter Month report for #{date_str}"
+
+        text_part do
+          body "Summary for #{date_str}\n
+                -------------------------------\n\n
+                Total kWH electricity consumed: #{consumption_month}\n
+                Total kWH electricity produced:  #{production_month}\n
+                Total m3 gas consumed: #{usage_month[:total_m3_gas_consumed]}\n\n
+                Levering kWH cost (Oxxio) : EUR #{ oxxio_raw_cost }\n
+                Levering kWH cost (EasyEnergy): EUR #{ easy_energy_raw_cost }\n
+                Total kWH cost (Oxxio) : EUR #{ oxxio_cost }\n
+                Total kWH cost (EasyEnergy): EUR #{ easy_energy_cost }, inclusief opslag van EUR #{ easy_energy_opslag }\n
+               "
+        end
+      
+        html_part do
+          content_type 'text/html; charset=UTF-8'
+          body "<h1>Summary for #{date_str}</h1>" +
+               "<p>Total kWH electricity consumed: #{consumption_month}</p>" +
+               "<p>Total kWH electricity produced:  #{production_month}</p>" +
+               "<p>Total m3 gas consumed: #{usage_month[:total_m3_gas_consumed]}</p>" +
+               "</br>" +
+               "<p>Levering kWH cost (Oxxio): <b>EUR #{ oxxio_raw_cost} </b></p>" +
+               "<p>Levering kWH cost (EasyEnergy): <b>EUR #{ easy_energy_raw_cost} </b></p>" +
+               "<p>Total kWH cost (Oxxio): <b>EUR #{ oxxio_cost} </b></p>" +
+               "<p>Total kWH cost (EasyEnergy): <b>EUR #{ easy_energy_cost} </b>, inclusief opslag van <b>EUR #{ easy_energy_opslag }</b></p>"
+        end 
+        
+      end
+      
+      mail.deliver!
+    end
+  end   
+end

app/helpers/SearchingForSyncState.rb

@@ -0,0 +1,13 @@
+class SearchingForSyncState < StatePattern::State
+  def handle_byte_stream(bytes)
+   idx = 0;
+   # spool unwanted bytes
+   while (idx < bytes.length && bytes[idx] != Synchronizer::SYNC_PATTERN[0]) do idx = idx+1 end
+   
+   #p "Found pattern at idx = #{idx}"
+   transition_to(ConfirmingSyncPatternState)
+   
+   # return
+   return bytes[idx..-1] || ""
+  end
+end

app/helpers/Synchronizer.rb

@@ -0,0 +1,9 @@
+class Synchronizer 
+  include StatePattern
+  
+  SYNC_PATTERN = "/CTA5ZIV\-METER\n\n"
+  #SYNC_PATTERN = "\n/ISk5\\2ME382-1003\n\n"
+
+  set_initial_state ::SearchingForSyncState
+
+end

app/helpers/TariffsMailer.rb

@@ -0,0 +1,46 @@
+require "mail"
+
+class TariffsMailer
+    
+  SSL_OPTS = {
+      :openssl_verify_mode => OpenSSL::SSL::VERIFY_NONE,
+  }  
+  
+                   
+  #
+  # Class methods
+  #
+  class << self
+    def deliver(date)     
+      # Read SMTP options from smtp.yml
+      smtp_opts = YAML::load(File.open('config/smtp.yml')).symbolize_keys
+      smtp_opts.merge!(SSL_OPTS) if smtp_opts[:ssl] && smtp_opts[:ssl_verify_mode].eql?("none")
+      
+      # Create png file
+      c = Cost.new
+      c.easy_energy_tariff_barplot(date)
+      
+      mail = Mail.new do
+        delivery_method :smtp, smtp_opts
+        to      ['Mannetje <aart@van-halteren.net>','Vrouwtje <irene@van-halteren.net>']
+        from    'SmartMeter <aart@van-halteren.net>'
+        subject "EasyEnergy tariffs for #{date}"
+
+        text_part do
+          body "Tariffs for #{date}\n"
+        end
+      
+        html_part do
+          content_type 'text/html; charset=UTF-8'
+          body "<h1>Tariffs for #{date} </h>"
+        end 
+        
+        # add attachment
+        filename = "easy_tariff_%s.png" % date.strftime("%F")
+        add_file :filename => filename, :content => File.read("plots/%s" % filename) 
+      end
+      
+      mail.deliver!
+    end
+  end
+end

app/models/battery.rb

@@ -0,0 +1,41 @@
+class Battery
+
+  attr_accessor :battery_kwh, :battery_max_kwh
+
+  def initialize(battery_capacity=10.0)
+    @battery_kwh = 0.0
+    @battery_max_kwh = battery_capacity
+  end
+  
+  def reset
+    @battery_kwh = 0.0
+  end
+  
+  def charge(kwh)
+    return 0.0 if kwh.nil?
+    if battery_kwh + kwh <= battery_max_kwh
+      @battery_kwh += kwh.to_f
+      #p "Battery is now at %s kwh" % battery_kwh
+      return kwh
+    else
+      old_kwh = battery_kwh
+      @battery_kwh = battery_max_kwh
+      #p "Battery is now at %s kwh" % battery_kwh
+      return (battery_max_kwh-old_kwh)
+    end
+  end
+    
+  def discharge(kwh)
+    return 0.0 if kwh.nil?
+    if battery_kwh > kwh.to_f
+      @battery_kwh -= kwh.to_f
+      #p "Battery is now at %s kwh" % battery_kwh
+      return kwh
+    else
+      old_kwh = battery_kwh
+      @battery_kwh = 0.0
+      #p "Battery is now at %s kwh" % battery_kwh
+      return old_kwh
+    end        
+  end  
+end

app/models/cost.rb

@@ -0,0 +1,459 @@
+require 'open-uri'
+require 'i18n'
+require 'gruff'
+
+EASY_ENERGY_TARIFFS = {}
+
+# See https://www.belastingdienst.nl/wps/wcm/connect/bldcontentnl/belastingdienst/zakelijk/overige_belastingen/belastingen_op_milieugrondslag/tarieven_milieubelastingen/tabellen_tarieven_milieubelastingen
+# Without VAT
+ENERGY_TAX_KWH = { 2020 => 0.09770, 2021 => 0.09428, 2022 => 0.03679, 2023 => 0.12599, 2024 => 0.10880, 2025 => 0.10154, 2026 => 0.09157 }
+ODE_KWH = { 2020 => 0.0273, 2021 => 0.0300, 2022 => 0.0305, 2023 => 0.0, 2024 => 0.0, 2025 => 0.0, 2026 =>0.0}
+# merge by adding values
+TAX_KWH = ENERGY_TAX_KWH.merge(ODE_KWH){|key, energy_tax, ode| energy_tax + ode}
+
+
+class Cost
+ 
+  attr_accessor :max_charge_kwh    
+  attr_reader :battery, :entsoe, :zone
+  
+  def initialize(zone="Amsterdam", battery_capacity=10.0, max_charge=5.0, storage_cost=0.05)
+    @zone = zone
+    @entsoe = Entsoe.new(zone, storage_cost)
+    @max_charge_kwh =  max_charge
+    @battery = Battery.new(battery_capacity)
+
+    # Allow for Dutch titles in graphs
+    I18n.load_path += Dir[File.expand_path("config/locales") + "/*.yml"]
+    I18n.available_locales = [:en, :nl]
+    I18n.locale = :nl
+  end
+  
+  # Government reduced VAT to 9% from 1 July 2022 until 31 Dec 2022
+  def vat_at(date)
+    jul22 = Date.parse("2022-09-01")
+    dec22 = Date.parse("2022-12-31")
+    if (date >= jul22 && date <= dec22)
+      0.09
+    else
+      0.21
+    end
+  end  
+  
+  def format_cost(cost)
+    cost ? "EUR %0.03f" % cost : "EUR ?"
+  end
+ 
+  # Assume: usage_kwh_cost, return_kwh_cost already includes vat
+  # calling this with only 'usage_kwh' parameter returns the tax amount for those kwh only.
+  # 
+  def add_tax(formatted_hour,usage_kwh,usage_kwh_cost=0.0,return_kwh=0.0, return_kwh_cost=0.0)
+    return nil if (usage_kwh.nil? || usage_kwh_cost.nil? || return_kwh.nil? || return_kwh_cost.nil?)
+    
+    vat = 1 + vat_at(Date.parse(formatted_hour))
+    year = Date.parse(formatted_hour).year
+    # calculate tax per kwh - which the sum of energy_tax and ode
+    tax = TAX_KWH[year]*vat
+    
+    usage_kwh * (usage_kwh_cost + tax) - return_kwh * (return_kwh_cost + tax)  
+    
+  end
+  
+  ######################################################
+  # Easy Energy - makes use of entsoe, adds EasyEnergy opslag
+  ######################################################
+  
+  def easy_energy_rate(formatted_hour)
+    year = Date.parse(formatted_hour).year
+    month = Date.parse(formatted_hour).month
+
+    case year
+    when 2020..2021
+      # opslag, zonder BTW
+      0.00800
+    when 2022
+      case month
+      when 1..11
+        # opslag, before increase
+        0.00800
+      when 12
+        # opslag, met BTW: 0,01962
+        0.018 
+      end
+    when 2023
+      0.018
+    when 2024..2026
+      # opslag met BTW: 0,02178
+      0.018457
+    end  
+  end
+  
+  def easy_energy_cost(formatted_hour, usage_kwh, return_kwh)
+    return nil if (usage_kwh.nil? || return_kwh.nil?)
+    #p "easy_energy_cost for " + formatted_hour
+
+    usage_kwh_cost = return_kwh_cost = (entsoe.price_at(formatted_hour)+easy_energy_rate(formatted_hour))*(1+vat_at(Date.parse(formatted_hour)))
+    add_tax(formatted_hour, usage_kwh, usage_kwh_cost, return_kwh, return_kwh_cost)
+  end
+  
+  def easy_energy_hours(date)
+    hour_start = date.in_time_zone(zone).beginning_of_day
+    day_end = hour_start.advance(days: 1)
+    result = []
+    while(hour_start < day_end) do
+       formatted_hour = hour_start.strftime("%F %H")
+       result << easy_energy_cost(formatted_hour, 1, 0)
+       hour_start = hour_start.advance(:hours => 1)
+    end
+    result
+  end
+  
+  def easy_energy_tariff_barplot(date)
+    hours = (0..23).to_a
+    costs = easy_energy_hours(date)
+    g = Gruff::Bar.new()
+    g.title = "Tarief per kwH (incl. belastingen en BTW) - %s" % I18n.localize(date, format: "%A, %e %B %Y")
+    g.x_axis_label = "uur"
+    #g.y_axis_label = "EUR"
+    g.y_axis_label_format = lambda do |value|
+      "€ %.2f" % value
+    end
+    g.labels = hours
+    g.data :costs, costs
+    g.write("plots/easy_tariff_%s.png" % date.strftime("%F"))
+  end
+
+  # calculate the hourly cost (raw!) between start_hour and end_hour
+  # raw means: entsoe prices + VAT
+  def easy_energy_raw_hourly_cost_between(start_hour, end_hour)
+    begin_hour = start_hour
+    costs = []
+    while(begin_hour < end_hour) do
+       # get usage_kwh/return_kwh for one hour
+       begin_hour_plus1 = begin_hour.end_of_hour
+       hour_readings = Reading.where("created_at > :begin AND created_at < :end", {:begin => begin_hour, :end => begin_hour_plus1})
+       hour_diff = hour_readings.last ? hour_readings.last.diff(hour_readings.first) : UNKNOWN_READING
+       # helper to remove distiction between low/high tarif consumption
+       usage_kwh, return_kwh = Reading.consumed_and_produced_for_diff(hour_diff)
+
+       formatted_hour = begin_hour.strftime("%F %H")
+       # calculate RAW price (entsoe price + VAT)
+       kwh_price = entsoe.price_at(formatted_hour)*(1+vat_at(Date.parse(formatted_hour)))
+       hour_cost = (usage_kwh.nil? || return_kwh.nil? || kwh_price.nil?) ? nil : ((usage_kwh-return_kwh)*kwh_price)
+       costs << hour_cost
+       
+        # do the next hour
+        begin_hour = begin_hour.advance(:hours => 1)
+    end
+    # give the result
+    costs
+  end  
+
+  # calculate the hourly cost between start_hour and end_hour
+  def easy_energy_hourly_cost_between(start_hour, end_hour)
+    begin_hour = start_hour
+    costs = []
+    while(begin_hour < end_hour) do
+       # get usage_kwh/return_kwh for one hour
+       begin_hour_plus1 = begin_hour.end_of_hour
+       hour_readings = Reading.where("created_at > :begin AND created_at < :end", {:begin => begin_hour, :end => begin_hour_plus1})
+       hour_diff = hour_readings.last ? hour_readings.last.diff(hour_readings.first) : UNKNOWN_READING
+       # helper to remove distiction between low/high tarif consumption
+       usage_kwh, return_kwh = Reading.consumed_and_produced_for_diff(hour_diff)
+
+       formatted_hour = begin_hour.strftime("%F %H")
+       costs << easy_energy_cost(formatted_hour, usage_kwh, return_kwh)
+       # do the next hour
+       begin_hour = begin_hour.advance(:hours => 1)
+    end
+    # give the result
+    costs
+  end
+
+  def easy_energy_daily_cost_between(start_day, end_day)
+    curr_day = start_day
+    costs = []
+    while (curr_day <= end_day) do
+      hour_start = curr_day.in_time_zone(zone).beginning_of_day
+      hour_end = hour_start.advance(days: 1)
+
+      costs_24hours = easy_energy_hourly_cost_between(hour_start,hour_end)
+      if costs_24hours.any?{ |e| e.nil? }
+        p "Not all Reading data between %s and %s is available!" % [I18n.localize(start_day, format: "%e %B %Y"), I18n.localize(end_day, format: "%e %B %Y")]
+      end   
+      # add the sum of 24 hours
+      costs << costs_24hours.compact.sum
+      # do the next day
+      curr_day = curr_day.advance(:days => 1)
+    end
+    # give the result
+    costs
+  end  
+
+  # calculate leveringskosten (entsoe prices + VAT) in one month
+  # date = arbitrary day for the month
+  def easy_energy_raw_cost_in_month(date)
+    hour_start = date.beginning_of_month.in_time_zone(zone).beginning_of_day
+    hour_end = date.end_of_month.in_time_zone(zone).end_of_day
+    # can't get cost for the future
+    hour_end = Time.now if hour_end > Time.now
+
+    easy_energy_raw_hourly_cost_between(hour_start,hour_end)
+  end  
+
+  def easy_energy_cost_in_month(date)
+    hour_start = date.beginning_of_month.in_time_zone(zone).beginning_of_day
+    hour_end = date.end_of_month.in_time_zone(zone).end_of_day
+    # can't get cost for the future
+    hour_end = Time.now if hour_end > Time.now
+
+    easy_energy_hourly_cost_between(hour_start,hour_end)
+  end
+
+  def easy_energy_hourly_cost_for(date)
+    hour_start = date.in_time_zone(zone).beginning_of_day
+    day_end = hour_start.advance(days: 1)
+    easy_energy_hourly_cost_between(hour_start, day_end)
+  end  
+  
+  def easy_energy_cost_barplot(date)
+    # get array with 24 hourly cost values
+    costs = easy_energy_hourly_cost_for(date)
+    
+    # create plot
+    hours = (0..23).to_a
+    g = Gruff::Bar.new
+    g.title = "Verbruikskosten (incl. belastingen en BTW) - %s" % I18n.localize(date, format: "%A, %e %B %Y")
+    g.x_axis_label = "uur"
+    #g.y_axis_label = "EUR"
+    g.y_axis_label_format = lambda do |value|
+      "€ %.2f" % value
+    end
+    g.labels = hours
+    g.data :costs, costs
+    g.write("plots/easy_cost_%s.png" % date.strftime("%F"))
+    
+    # return the sum cost
+    costs.sum
+  end
+  
+
+  ######################################################
+  # Oxxio rates and cost
+  ######################################################
+  
+  def oxxio_rate(formatted_hour,  high_tariff)
+    year = Date.parse(formatted_hour).year
+    case year
+      when 2020
+        high_tariff ? 0.07865 : 0.06215
+      when 2021
+        high_tariff ? 0.06782 : 0.05259
+      when 2022
+        high_tariff ? 0.23665 : 0.19408
+      when 2023
+        # rate excl. VAT
+        high_tariff ? 0.47758 : 0.34165  
+      when 2024
+        0.25767769
+      when 2025
+        high_tariff ? 0.2695 : 0.2296
+      when 2026
+        high_tariff ? 0.23186 : 0.22442
+    end
+  end
+  
+  # Optional: raw! (tarif + VAT) cost
+  def oxxio_energy_cost(formatted_hour, normaal_kwh, dal_kwh, with_tax=true, year_shift=0)
+    return nil if (normaal_kwh.nil? || dal_kwh.nil?)
+    #year = Date.parse(formatted_hour).year+year_shift
+    date = Date.parse(formatted_hour).advance(years: year_shift)
+    case date.to_time.to_i
+      # Date.parse("2019-12-08").to_time.to_i
+      # From 8 Dec 2019 until 7 Dec 2020
+      when 1575763200..1607385599
+         #normaal_kwh * (0.07865 + 0.11822 + 0.03303) + dal_kwh * (0.06215 + 0.11822 + 0.03303) 
+         normaal_kwh_cost = 0.07865
+         dal_kwh_cost = 0.06215
+      # From 8 Dec 2020 until 7 Dec 2021
+      when 1607385600..1638921599
+         #normaal_kwh * (0.06782 + 0.11408 + 0.03630) + dal_kwh * (0.05259 + 0.11408 + 0.03630) 
+         normaal_kwh_cost = 0.06782
+         dal_kwh_cost = 0.05259
+      #  From 8 Dec 2021 until 7 Sept 2022
+      when 1638921600..1662595199
+        # normaal_kwh * (0.23665 + 0.04452 + 0.03691) + dal_kwh * (0.19408 + 0.04452 + 0.03691) 
+        normaal_kwh_cost = 0.23665
+        dal_kwh_cost = 0.19408
+      # From 8 Sept 2022 until 31 December 2022 
+      when 1662595200..1672527599
+        normaal_kwh_cost = 0.60824
+        dal_kwh_cost = 0.43701
+      # From 1 Jan 2023 until 31 December 2023
+      when 1672527600..1704063599
+        vat = 1 + vat_at(Date.parse(formatted_hour))
+        normaal_kwh_cost = 0.47758*vat
+        dal_kwh_cost = 0.34165*vat   
+      # From 1 Jan 2024 until 31 December 2024
+      when 1704063600..1735603199
+        vat = 1 + vat_at(Date.parse(formatted_hour))
+        normaal_kwh_cost = 0.25767769*vat
+        dal_kwh_cost = 0.25767769*vat  
+      when 1735711200..1767160800
+        vat = 1 + vat_at(Date.parse(formatted_hour))
+        normaal_kwh_cost = 0.2695*vat
+        dal_kwh_cost = 0.2296*vat
+      when 1767225600..1785887999  # 2026 full year
+        normaal_kwh_cost = 0.19161
+        dal_kwh_cost = 0.18547
+      else
+        p "Not supported interval Oxxio for value: %d" % date.to_time.to_i
+        # catch-all, incase 'formated_hour' is outside any of the cases
+        normaal_kwh_cost = 0.0
+        dal_kwh_cost = 0.0
+    end
+    
+    if with_tax
+      normaal_cost = add_tax(formatted_hour, normaal_kwh,normaal_kwh_cost) # return_kwh already accounted for
+      dal_cost = add_tax(formatted_hour, dal_kwh, dal_kwh_cost)
+    else
+      normaal_cost = normaal_kwh*normaal_kwh_cost
+      dal_cost = dal_kwh*dal_kwh_cost
+    end
+
+    # result
+    return normaal_cost, dal_cost
+  end  
+
+  #
+  # Entsoe
+  #
+  
+  def entsoe_energy_cost(formatted_hour, usage_kwh, return_kwh)
+    return nil if (usage_kwh.nil? || return_kwh.nil?)
+    
+    usage_kwh_cost = return_kwh_cost = entsoe.price_at(formatted_hour)*(1+vat_at(Date.parse(formatted_hour)))
+    add_tax(formatted_hour, usage_kwh, usage_kwh_cost, return_kwh, return_kwh_cost)
+  end
+  
+  ######################################################
+  #  Calculate the per_hour usage and costs
+  ######################################################
+  
+  def hours(date, year_shift=0)
+    hour_start = date.in_time_zone(zone).beginning_of_day
+    day_end = hour_start.advance(days: 1)
+    result = []
+    lowest_hour, highest_hour,high_hours = entsoe.high_low_hours(date.advance(years: year_shift))
+    
+    while(hour_start < day_end) do
+      hour_end = hour_start.end_of_hour
+      
+      #p "Fetching meter readings between %s and %s" % [hour_start,hour_end]
+      hour_readings = Reading.where("created_at > :begin AND created_at < :end", {:begin => hour_start, :end => hour_end})
+      hour_diff = hour_readings.last ? hour_readings.last.diff(hour_readings.first) : UNKNOWN_READING
+      # calculate cost of this hour
+      usage_kwh = hour_diff[:total_kwh_consumed_high] + hour_diff[:total_kwh_consumed_low] rescue nil
+      return_kwh = hour_diff[:total_kwh_produced_high] + hour_diff[:total_kwh_produced_low] rescue nil
+
+      formatted_hour = hour_start.advance(years: year_shift).strftime("%F %H")
+      easy_cost = easy_energy_cost(formatted_hour, usage_kwh, return_kwh) # without battery use
+
+      #
+      # Make battery work
+      #     
+      if !usage_kwh.nil?
+        # charge battery with return_kwh
+        return_kwh -= battery.charge(return_kwh)
+        
+        if (lowest_hour.eql?(formatted_hour) || entsoe.price_at(formatted_hour) < 0) # lowest_hour = "" if small difference between high/low
+          # charge battery during lowest hour, or when prices are negative
+          usage_kwh += battery.charge(max_charge_kwh)
+        else
+          # if during expensive hours || more than <max_charge_kwh> kwh then discharge_battery
+          # if (battery.battery_kwh > max_charge_kwh) || high_hours.include?(formatted_hour)
+          if high_hours.include?(formatted_hour)
+             usage_kwh -= battery.discharge(usage_kwh)
+          end   
+        end    
+      end       
+      easy_cost_with_battery = easy_energy_cost(formatted_hour, usage_kwh, return_kwh) # with battery use
+            
+      #
+      # end of battery work
+      #
+      
+      normal_usage_kwh = hour_diff[:total_kwh_consumed_low] - hour_diff[:total_kwh_produced_low] rescue nil
+      low_usage_kwh = hour_diff[:total_kwh_consumed_high] - hour_diff[:total_kwh_produced_high] rescue nil
+      oxxio_cost = oxxio_energy_cost(formatted_hour,normal_usage_kwh, low_usage_kwh, year_shift)
+      oxxio_rate = oxxio_rate(formatted_hour, (hour_readings.first.high_tarif rescue true))
+      
+      one_hour = [formatted_hour, 
+                  hour_diff[:total_kwh_consumed_high], 
+                  hour_diff[:total_kwh_consumed_low], 
+                  hour_diff[:total_kwh_produced_high], 
+                  hour_diff[:total_kwh_produced_low],
+                  usage_kwh,
+                  return_kwh,
+                  battery.battery_kwh,
+                  entsoe.price_at(formatted_hour), # entsoe rate
+                  easy_cost_with_battery,
+                  easy_cost,
+                  oxxio_rate, 
+                  oxxio_cost]
+      
+      p "%s,%s,%s,%s,%s,%s,%s,%0.1f,%s,%s,%s,%s,%s" % (one_hour[0..7] + one_hour[8..12].map{|c| format_cost(c)})
+      result << one_hour
+
+      hour_start = hour_start.advance(:hours => 1)
+    end
+    result
+  end
+  
+  def summarize_hours(hours)
+    usage_kwh = hours.map{|e| (e[1] && e[2]) ? (e[1] + e[2]) : nil}.compact.sum
+    return_kwh = hours.map{|e| (e[3] && e[4]) ? (e[3] + e[4]) : nil}.compact.sum
+    usage_kwh_with_battery = hours.map{|e| e[5]}.compact.sum
+    return_kwh_with_battery = hours.map{|e| e[6]}.compact.sum
+    entsoe_cost_with_battery = hours.map{|e| e[9]}.compact.sum
+    entsoe_cost = hours.map{|e| e[10]}.compact.sum
+    oxxio_cost = hours.map{|e| e[12]}.compact.sum
+    
+    return usage_kwh, return_kwh, usage_kwh_with_battery, return_kwh_with_battery, format_cost(entsoe_cost_with_battery), format_cost(entsoe_cost), format_cost(oxxio_cost)
+  end
+  
+  def hours_in_month(date, year_shift=0)
+    day = date.beginning_of_month
+    last_day = date.end_of_month
+    all_hours = []
+    while (day <= last_day)
+      all_hours += hours(day, year_shift)
+      day = day.advance(:days => 1)
+    end
+    all_hours
+  end
+  
+  # heel2021 = hours_in(Date.parse("2021-01-01"), Date.parse("2021-12-31"), 1)
+  
+  def hours_in(from,to, year_shift=0)
+    day = from.to_date
+    last_day = to.to_date
+    all_hours = []
+    while (day <= last_day)
+      all_hours += hours(day, year_shift)
+      day = day.advance(:days => 1)
+    end
+    all_hours
+  end
+  
+  def write_csv(filename,hours)
+    CSV.open(filename, "wb") do |csv|
+      csv << ["hour", "kwh_consumed_low", "kwh_consumed_high", "kwh_produced_low", "kwh_produced_high", "easy_usage_rate", "easy_return_rate", "easy_cost", "oxxio_rate", "oxxio_cost"]
+      hours.each do |row| 
+        csv << row
+      end  
+    end
+  end
+    
+
+end

app/models/entsoe.rb

@@ -0,0 +1,166 @@
+#
+# Obtain energy prices from Entsoe (transparancy platform)
+# See https://transparency.entsoe.eu/content/static_content/Static%20content/web%20api/Guide.html
+#
+
+require 'open-uri'
+require 'nokogiri'
+
+class Entsoe 
+
+  URL = 'https://web-api.tp.entsoe.eu'
+  
+  attr_accessor :no_grid_charge_months, :zone
+  attr_reader :storage_cost
+
+  def initialize(zone = "Amsterdam", storage_cost = 0.05, api_key = "c2287e07-0c26-4950-b430-22b7f75a8f2e")
+    @api_key = api_key
+    @kwh_prices = {}  
+    @storage_cost = storage_cost # how much does it cost to store 1 kwh in battery
+    @no_grid_charge_months = [5,6,7] # months where we assume there is enough surplus from sun power
+    @zone = zone
+  end
+  
+  def price_at(formatted_hour)
+    unless @kwh_prices.key?(formatted_hour)
+      p "Fetching Entsoe tariffs for %s" % formatted_hour
+      prices = query_day_ahead_prices(Date.parse(formatted_hour))
+      @kwh_prices.merge!(hourly_format(prices))
+    end
+    @kwh_prices[formatted_hour]
+  end
+  
+  def prices_at(date)
+    hour_start = date.beginning_of_day.in_time_zone(@zone)
+    day_end = hour_start.advance(days: 1)
+    result = {}
+    while(hour_start < day_end) do
+        formatted_hour = hour_start.strftime("%F %H")
+        result.merge!({ formatted_hour => price_at(formatted_hour)})
+        hour_start = hour_start.advance(:hours => 1)
+    end
+    result    
+  end
+  
+  def high_low_hours(date)
+    sorted_prices = prices_at(date).to_a.sort_by(&:last) # sort according to price 
+    highest_hour = sorted_prices.last[0] 
+    
+    # Some months: do not charge; every hour that has cost > 0 is a high_hour
+    if @no_grid_charge_months.include?(date.month)
+      lowest_hour = "" # effectively no charging from grid
+      high_hours = sorted_prices.select{|p| p[1] > 0}.to_h.keys
+    else      
+      lowest_hour = sorted_prices.first[0]
+      
+      # calculate hours where rate > charge_rate + storage_cost (typically 0.05)
+      charge_rate = sorted_prices.first[1] # assume we charge at lowest hour
+      high_hours = sorted_prices.select{|p| p[1] > charge_rate + storage_cost}.to_h.keys
+      
+      # reset lowest_hour (effectively not charging), when price difference smaller than storage_cost
+      highest_price  = sorted_prices.last[1]
+      lowest_hour = "" if (highest_price-charge_rate) < storage_cost
+    end
+    
+    return lowest_hour,highest_hour,high_hours
+  end
+  
+  def query_day_ahead_prices(date)
+    start_date = date.beginning_of_day
+    end_date = date.end_of_day.advance(hours: -1)
+    
+    # A44 - Document type => Price Document
+    # NL =  '10YNL----------L'
+    
+    domain = '10YNL----------L'      
+    url = URL + "/api?securityToken=%s&documentType=A44&in_Domain=%s&out_Domain=%s&periodStart=%s&periodEnd=%s" % [@api_key, domain, domain, start_date.strftime("%Y%m%d%H%M"), end_date.strftime("%Y%m%d%H00")]
+    #p url
+    
+    base_request(date, url)
+  end
+  
+ 
+  private
+  
+    def hourly_format(prices)
+      # in memory hash @kwh_prices is in @zone timezone
+      prices.to_a.map{|p| [p[0].in_time_zone(zone).strftime("%F %H"),p[1]]}.to_h
+    end
+  
+    # get position and amount from XML snippet
+    # <Point>
+    #  <position>1</position>
+    #  <price.amount>196.23</price.amount>
+    # </Point>
+    #
+    # convert price to EUR per kwh, excluding VAT
+    #
+    def parse_point(xml)
+      return xml.xpath(".//xmlns:position").text.to_i, (xml.xpath(".//xmlns:price.amount").text.to_f/1000).round(5)
+    end
+  
+    def base_request(date, url)
+      formatted_date = date.strftime("%F")
+      
+      doc = Nokogiri::XML(URI.open(url))
+
+      #p "Entsoe prices: %s" % doc
+      
+      prices = doc.xpath('.//xmlns:Point').map{|p| parse_point(p)}
+      begin
+        # get start_time (in UTC) from XML docment
+        start_time = DateTime.parse(doc.xpath('.//xmlns:period.timeInterval//xmlns:start').text)
+        
+        # Get resolution to determine expected number of positions
+        resolution = doc.xpath('.//xmlns:resolution').text
+        interval_minutes = resolution == 'PT15M' ? 15 : 60
+        expected_positions = (24 * 60) / interval_minutes
+        
+        # Create hash from available prices
+        price_hash = prices.to_h
+        
+        # Fill in missing positions by interpolating from adjacent values
+        complete_prices = {}
+        (1..expected_positions).each do |position|
+          if price_hash.key?(position)
+            complete_prices[position] = price_hash[position]
+          else
+            # Find previous and next available prices for interpolation
+            prev_price = (position-1).downto(1).find { |p| price_hash.key?(p) }
+            next_price = (position+1).upto(expected_positions).find { |p| price_hash.key?(p) }
+            
+            if prev_price && next_price
+              # Interpolate between previous and next
+              complete_prices[position] = ((price_hash[prev_price] + price_hash[next_price]) / 2.0).round(5)
+            elsif prev_price
+              # Use previous price as fallback
+              complete_prices[position] = price_hash[prev_price]
+            elsif next_price
+              # Use next price as fallback
+              complete_prices[position] = price_hash[next_price]
+            end
+            
+            # Calculate the formatted hour for the warning message
+            hour_offset = interval_minutes == 15 ? (position - 1) / 4 : (position - 1)
+            missing_hour = start_time.advance(hours: hour_offset).in_time_zone(zone).strftime("%F %H")
+            p "Warning: Missing Entsoe data for #{missing_hour}, interpolated value: #{complete_prices[position]}"
+          end
+        end
+        
+        #returns a hash with keys formatted "yyyy-mm-dd hr" and values price (per kwh)
+        # <position> tag runs from 1-96 for 15min intervals. Convert to hourly by taking first interval of each hour
+        if interval_minutes == 15
+          # For 15-minute intervals, use the first interval of each hour (positions 1, 5, 9, 13, ...)
+          complete_prices.select { |pos, _| (pos - 1) % 4 == 0 }
+                        .map { |pos, price| [start_time.advance(hours: ((pos - 1) / 4)), price] }
+                        .to_h
+        else
+          # For hourly data (position runs from 1-24, we need hours from 00-23)
+          complete_prices.map { |pos, price| [start_time.advance(hours: (pos - 1)), price] }.to_h
+        end
+      rescue Date::Error => e
+        p e.message
+        {}
+      end    
+    end
+end

app/models/reading.rb

@@ -0,0 +1,151 @@
+
+UNKNOWN_READING = { :total_kwh_consumed_high => nil, :total_kwh_consumed_low => nil, :total_kwh_produced_high => nil, :total_kwh_produced_low => nil, :total_m3_gas_consumed => nil}
+  
+
+class Reading < ActiveRecord::Base
+   
+  # Round up with 1 (default) decimals 
+  def round_up(number, decimals = 1)
+    factor = 10 ** decimals
+    (number * factor).ceil / factor.to_f
+  end
+
+
+  def eql_reading?(reading)
+    self.total_kwh_consumed_high == reading.total_kwh_consumed_high &&
+    self.total_kwh_consumed_low == reading.total_kwh_consumed_low &&
+    self.total_kwh_produced_high == reading.total_kwh_produced_high &&
+    self.total_kwh_produced_low == reading.total_kwh_produced_low &&
+    round_up(self.current_kw_consumed) == round_up(reading.current_kw_consumed) &&
+    round_up(self.current_kw_produced) == round_up(reading.current_kw_produced) &&
+    self.total_m3_gas_consumed == reading.total_m3_gas_consumed &&
+    self.high_tarif == reading.high_tarif
+  end
+  
+  # reduce precision to 1 digit behind comma
+  def total_kwh_consumed_high=(kwh)
+    write_attribute(:total_kwh_consumed_high,kwh.round(1))
+  end
+
+  # reduce precision to 1 digit behind comma
+  def total_kwh_consumed_low=(kwh)
+    write_attribute(:total_kwh_consumed_low,kwh.round(1))
+  end
+
+  # reduce precision to 1 digit behind comma
+  def total_kwh_produced_high=(kwh)
+    write_attribute(:total_kwh_produced_high,kwh.round(1))
+  end
+
+  # reduce precision to 1 digit behind comma
+  def total_kwh_produced_low=(kwh)
+    write_attribute(:total_kwh_produced_low,kwh.round(1))
+  end
+ 
+  # calculate difference with another reading
+  # return a hash with differences (self - reading)
+  def diff(reading)
+    if reading
+      { :total_kwh_consumed_high => (self.total_kwh_consumed_high - reading.total_kwh_consumed_high).round(1),
+        :total_kwh_consumed_low => (self.total_kwh_consumed_low - reading.total_kwh_consumed_low).round(1),
+        :total_kwh_produced_high => (self.total_kwh_produced_high - reading.total_kwh_produced_high).round(1),
+        :total_kwh_produced_low => (self.total_kwh_produced_low - reading.total_kwh_produced_low).round(1),
+        :total_m3_gas_consumed => (self.total_m3_gas_consumed - reading.total_m3_gas_consumed).round(3) }
+    else
+      { :total_kwh_consumed_high => self.total_kwh_consumed_high,
+        :total_kwh_consumed_low => self.total_kwh_consumed_low,
+        :total_kwh_produced_high => self.total_kwh_produced_high,
+        :total_kwh_produced_low => self.total_kwh_produced_low,
+        :total_m3_gas_consumed => self.total_m3_gas_consumed }
+    end
+  end
+  
+  
+  
+  #
+  # Class methods
+  #
+  class << self
+    # return readings from beginning of 'from' until end of 'to'
+    def days(from, to)
+      Reading.where("created_at > :begin AND created_at < :end", { :begin => from.to_date.beginning_of_day, :end => to.to_date.end_of_day})  
+    end
+    
+    def day(date)
+      Reading.where("created_at > :begin AND created_at < :end", { :begin => date.to_date.beginning_of_day, :end => date.to_date.end_of_day})        
+    end
+
+    def in_month(date)
+      hour_start = date.beginning_of_month.beginning_of_day
+      hour_end = date.end_of_month.end_of_day
+      # can't get cost for the future
+      hour_end = Time.now if hour_end > Time.now
+
+      Reading.where("created_at > :begin AND created_at < :end", { :begin => hour_start, :end => hour_end})  
+
+    end  
+    
+    def max_charge_kwh=(kwh)
+      @@max_charge_kwh = kwh
+    end
+    
+    def max_charge_kwh
+      @@max_charge_kwh
+    end
+    
+    # do not make distinction between high and low consumption/production
+    def consumed_and_produced_for_diff(d)
+      usage_kwh = (d[:total_kwh_consumed_high].nil? || d[:total_kwh_consumed_low].nil?) ? nil :  d[:total_kwh_consumed_high] + d[:total_kwh_consumed_low] 
+      return_kwh = (d[:total_kwh_produced_high].nil? || d[:total_kwh_produced_low].nil?) ? nil : d[:total_kwh_produced_high] + d[:total_kwh_produced_low]
+
+      return usage_kwh, return_kwh
+    end
+
+    # calculate net_consumption (= consumption-production)
+    def net_consumed_high_and_low_for_diff(d)
+      net_consumed_high = (d[:total_kwh_consumed_high].nil? || d[:total_kwh_produced_high].nil?) ? nil : d[:total_kwh_consumed_high]-d[:total_kwh_produced_high]
+      net_consumed_low = (d[:total_kwh_consumed_low].nil? || d[:total_kwh_produced_low].nil?) ? nil : d[:total_kwh_consumed_low]-d[:total_kwh_produced_low]
+
+      return net_consumed_high, net_consumed_low
+    end  
+    
+    def diff_on(date)
+      readings_on = day(date)
+      first = readings_on.first
+      last = readings_on.last
+      if last
+        last.diff(first)
+      else
+        { :total_kwh_consumed_high => 0, :total_kwh_consumed_low => 0, :total_kwh_produced_high => 0, :total_kwh_produced_low => 0, :total_m3_gas_consumed => 0 }
+      end
+    end
+
+    def diff_between(from_date, to_date)
+      readings_on_days = days(from_date, to_date)
+      first = readings_on_days.first
+      last = readings_on_days.last
+      if last
+        last.diff(first)
+      else
+        { :total_kwh_consumed_high => 0, :total_kwh_consumed_low => 0, :total_kwh_produced_high => 0, :total_kwh_produced_low => 0, :total_m3_gas_consumed => 0 }
+      end
+    end
+
+    def diff_in_month(date)
+      hour_start = date.beginning_of_month.beginning_of_day
+      hour_end = date.end_of_month.end_of_day
+      # can't get cost for the future
+      hour_end = Time.now if hour_end > Time.now
+
+      readings_in_month =  Reading.where("created_at > :begin AND created_at < :end", { :begin => hour_start, :end => hour_end})  
+      last = readings_in_month.last
+      if last
+        last.diff(readings_in_month.first)
+      else
+        { :total_kwh_consumed_high => 0, :total_kwh_consumed_low => 0, :total_kwh_produced_high => 0, :total_kwh_produced_low => 0, :total_m3_gas_consumed => 0 }
+      end
+
+    end  
+
+  end
+end

ar-no-rails.rb

@@ -0,0 +1,16 @@
+require "rubygems"
+require "bundler/setup"
+require "active_record"
+require "open-uri"
+#require 'gr/plot'
+#require 'histogram'
+
+project_root = File.dirname(File.absolute_path(__FILE__))
+Dir.glob(project_root + "/app/models/*.rb").each{|f| require f}
+
+connection_details = YAML::load(File.open('config/database.yml'))
+ActiveRecord::Base.establish_connection(connection_details)
+
+if __FILE__ == $0
+  puts "Count of Pages: #{Page.count}"
+end

config/database.yml

@@ -0,0 +1,6 @@
+host: 'smartmeter_db'
+adapter: 'mysql2'
+database: 'smartmeter'
+username: 'root' 
+password: 'rootme'
+pool: 5

config/locales/nl.yml

@@ -0,0 +1,213 @@
+---
+nl:
+  activerecord:
+    errors:
+      messages:
+        record_invalid: 'Validatie mislukt: %{errors}'
+        restrict_dependent_destroy:
+          has_one: Kan item niet verwijderen omdat %{record} afhankelijk is
+          has_many: Kan item niet verwijderen omdat afhankelijke %{record} bestaan
+  date:
+    abbr_day_names:
+    - zo
+    - ma
+    - di
+    - wo
+    - do
+    - vr
+    - za
+    abbr_month_names:
+    - 
+    - jan
+    - feb
+    - mrt
+    - apr
+    - mei
+    - jun
+    - jul
+    - aug
+    - sep
+    - okt
+    - nov
+    - dec
+    day_names:
+    - zondag
+    - maandag
+    - dinsdag
+    - woensdag
+    - donderdag
+    - vrijdag
+    - zaterdag
+    formats:
+      default: "%d-%m-%Y"
+      long: "%e %B %Y"
+      short: "%e %b"
+    month_names:
+    - 
+    - januari
+    - februari
+    - maart
+    - april
+    - mei
+    - juni
+    - juli
+    - augustus
+    - september
+    - oktober
+    - november
+    - december
+    order:
+    - :day
+    - :month
+    - :year
+  datetime:
+    distance_in_words:
+      about_x_hours:
+        one: ongeveer een uur
+        other: ongeveer %{count} uur
+      about_x_months:
+        one: ongeveer een maand
+        other: ongeveer %{count} maanden
+      about_x_years:
+        one: ongeveer een jaar
+        other: ongeveer %{count} jaar
+      almost_x_years:
+        one: bijna een jaar
+        other: bijna %{count} jaar
+      half_a_minute: een halve minuut
+      less_than_x_seconds:
+        one: minder dan een seconde
+        other: minder dan %{count} seconden
+      less_than_x_minutes:
+        one: minder dan een minuut
+        other: minder dan %{count} minuten
+      over_x_years:
+        one: meer dan een jaar
+        other: meer dan %{count} jaar
+      x_seconds:
+        one: "%{count} seconde"
+        other: "%{count} seconden"
+      x_minutes:
+        one: "%{count} minuut"
+        other: "%{count} minuten"
+      x_days:
+        one: "%{count} dag"
+        other: "%{count} dagen"
+      x_months:
+        one: "%{count} maand"
+        other: "%{count} maanden"
+      x_years:
+        one: "%{count} jaar"
+        other: "%{count} jaar"
+    prompts:
+      second: seconde
+      minute: minuut
+      hour: uur
+      day: dag
+      month: maand
+      year: jaar
+  errors:
+    format: "%{attribute} %{message}"
+    messages:
+      accepted: moet worden geaccepteerd
+      blank: moet opgegeven zijn
+      confirmation: komt niet overeen met %{attribute}
+      empty: moet opgegeven zijn
+      equal_to: moet gelijk zijn aan %{count}
+      even: moet even zijn
+      exclusion: is gereserveerd
+      greater_than: moet groter zijn dan %{count}
+      greater_than_or_equal_to: moet groter dan of gelijk zijn aan %{count}
+      inclusion: is niet in de lijst opgenomen
+      invalid: is ongeldig
+      less_than: moet minder zijn dan %{count}
+      less_than_or_equal_to: moet minder dan of gelijk zijn aan %{count}
+      model_invalid: 'Validatie mislukt: %{errors}'
+      not_a_number: is geen getal
+      not_an_integer: moet een geheel getal zijn
+      odd: moet oneven zijn
+      other_than: moet anders zijn dan %{count}
+      present: moet leeg zijn
+      required: moet bestaan
+      taken: is al in gebruik
+      too_long:
+        one: is te lang (maximaal %{count} teken)
+        other: is te lang (maximaal %{count} tekens)
+      too_short:
+        one: is te kort (minimaal %{count} teken)
+        other: is te kort (minimaal %{count} tekens)
+      wrong_length:
+        one: heeft onjuiste lengte (moet %{count} teken lang zijn)
+        other: heeft onjuiste lengte (moet %{count} tekens lang zijn)
+    template:
+      body: 'Er zijn problemen met de volgende velden:'
+      header:
+        one: "%{model} niet opgeslagen: %{count} fout gevonden"
+        other: "%{model} niet opgeslagen: %{count} fouten gevonden"
+  helpers:
+    select:
+      prompt: Maak een keuze
+    submit:
+      create: "%{model} toevoegen"
+      submit: "%{model} opslaan"
+      update: "%{model} bijwerken"
+  number:
+    currency:
+      format:
+        delimiter: "."
+        format: "%u %n"
+        precision: 2
+        separator: ","
+        significant: false
+        strip_insignificant_zeros: false
+        unit: "€"
+    format:
+      delimiter: "."
+      precision: 2
+      separator: ","
+      significant: false
+      strip_insignificant_zeros: false
+    human:
+      decimal_units:
+        format: "%n %u"
+        units:
+          billion: miljard
+          million: miljoen
+          quadrillion: biljard
+          thousand: duizend
+          trillion: biljoen
+          unit: ''
+      format:
+        delimiter: ''
+        precision: 3
+        significant: true
+        strip_insignificant_zeros: true
+      storage_units:
+        format: "%n %u"
+        units:
+          byte:
+            one: byte
+            other: bytes
+          gb: GB
+          kb: KB
+          mb: MB
+          tb: TB
+    percentage:
+      format:
+        delimiter: ''
+        format: "%n%"
+    precision:
+      format:
+        delimiter: ''
+  support:
+    array:
+      last_word_connector: " en "
+      two_words_connector: " en "
+      words_connector: ", "
+  time:
+    am: "'s ochtends"
+    formats:
+      default: "%a %d %b %Y %H:%M:%S %Z"
+      long: "%d %B %Y %H:%M"
+      short: "%d %b %H:%M"
+    pm: "'s middags"

config/smtp.yml

@@ -0,0 +1,8 @@
+address: "mail.van-halteren.net"
+port: 465
+domain: 'van-halteren.net'
+user_name: 'aart@van-halteren.net'
+password: 'secret'
+authentication: "plain"
+ssl: true
+ssl_verify_mode: "none"

daemonize.rb

@@ -0,0 +1,9 @@
+#require 'rubygems'
+require 'daemons'
+ 
+#pwd = Dir.pwd
+pwd = '/home/pcog/smartmeter' 
+Daemons.run_proc('smartmeter', {:dir_mode => :normal, :dir => pwd+"/pids"}) do
+  Dir.chdir(pwd)
+  exec "ruby smartmeter.rb"
+end

db/migrate/001_creates_readings.rb

@@ -0,0 +1,15 @@
+class CreatesReadings < ActiveRecord::Migration[4.2]
+  def change
+    create_table :readings do |t|
+      t.float :total_kwh_consumed_high
+      t.float :total_kwh_consumed_low
+      t.float :total_kwh_produced_high
+      t.float :total_kwh_produced_low
+      t.float :current_kw_consumed
+      t.float :current_kw_produced
+      t.float :total_m3_gas_consumed
+      t.boolean :high_tarif
+      t.timestamps
+    end
+  end
+end

db/migrate/002_add_created_at_index_to_readings.rb

@@ -0,0 +1,5 @@
+class AddCreatedAtIndexToReadings < ActiveRecord::Migration[4.2]
+  def change
+     add_index :readings, :created_at
+  end
+end

db/migrate/003_creates_prices.rb

@@ -0,0 +1,11 @@
+class CreatesPrices < ActiveRecord::Migration[4.2]
+  def change
+    create_table :prices do |t|
+      t.datetime :hour
+      t.float :usage_kwh
+      t.timestamps
+    end  
+    
+    add_index :prices, :hour 
+  end
+end

docker-compose.yml

@@ -0,0 +1,35 @@
+services:
+  db:
+    container_name: smartmeter_db
+    restart: unless-stopped
+    image: mysql:8.3
+    volumes:
+      - $PWD/data:/var/lib/mysql
+    ports:
+      - 3306:3306
+    environment:
+      MYSQL_ROOT_PASSWORD: rootme
+      MYSQL_DATABASE: smartmeter
+  smartmeter:
+    container_name: smartmeter
+    restart: unless-stopped
+    build: .
+    command: 'ruby ./smartmeter.rb'
+    #devices:
+    #  - "/dev/ttyUSB1:/dev/ttyUSB0"
+    volumes:
+      - .:/usr/src/app
+    depends_on:
+      - db
+  rstudio:
+    container_name: rstudio
+    restart: unless-stopped
+    build: ./rstudio
+    environment:
+      PASSWORD: secret
+    volumes:
+      - ./rstudio:/home/rstudio/smartmeter
+    ports:
+      - 8787:8787
+
+

etc/daily_mailer

@@ -0,0 +1,13 @@
+#
+# Not used. We now call docker from crontab
+#  /usr/bin/docker run --network smartmeter_default --rm smartmeter_smartmeter ruby report_mailer.rb
+#
+
+#!/usr/bin/env bash
+
+# load rvm ruby
+source /usr/local/rvm/environments/ruby-2.4.1@smartmeter
+
+cd /home/pcog/smartmeter
+ruby report_mailer.rb
+

etc/smartmeter

@@ -0,0 +1,20 @@
+#!/bin/bash
+#
+# Startup script (put in /etc/init.d/smartmeter)
+#
+# description: Starts Smartmeter as an unprivileged user.
+#
+
+# Create a wrapper using 'rvm alias smartmeter ruby-1.9.3-p484@smartmeter'
+
+if [[ -s "/usr/local/rvm/wrappers/smartmeter/ruby" ]]
+then
+  sudo -u www-data /usr/local/rvm/wrappers/smartmeter/ruby /home/pcog/smartmeter/daemonize.rb $1
+  RETVAL=$?
+
+  exit $RETVAL
+else
+  echo "ERROR: Missing RVM wrapper file: '/usr/local/rvm/wrappers/smartmeter'" >&2
+  exit 1
+fi
+

example_blurp.txt

@@ -0,0 +1,185 @@
+3233)
+1-0:1.8.1(001402.671*kWh)
+1-0:1.8.2(001130.208*kWh)
+1-0:2.8.1(000080.565*kWh)
+1-0:2.8.2(000205.556*kWh)
+0-0:96.14.0(0001)
+1-0:1.7.0(01.283*kW)
+1-0:2.7.0(00.000*kW)
+0-0:96.7.21(00015)
+0-0:96.7.9(00004)
+1-0:99.97.0(0)(0-0:96.7.19)
+1-0:32.32.0(00000)
+1-0:32.36.0(00003)
+0-0:96.13.0()
+1-0:32.7.0(236.0*V)
+1-0:31.7.0(005*A)
+1-0:21.7.0(01.283*kW)
+1-0:22.7.0(00.000*kW)
+0-1:24.1.0(003)
+0-1:96.1.0(4730303933303034333734333338333235)
+0-1:24.2.1(260103201000W)(00272.130*m3)
+!7603
+/CTA5ZIV-METER
+
+1-3:2.671*kWh)
+1-0:1.8.2(001130.208*kWh)
+1-0:2.8.1(000080.565*kWh)
+1-0:2.8.2(000205.556*kWh)
+0-0:96.14.0(0001)
+1-0:1.7.0(01.286*kW)
+1-0:2.7.0(00.000*kW)
+0-0:96.7.21(00015)
+0-0:96.7.9(00004)
+1-0:99.97.0(0)(0-0:96.7.19)
+1-0:32.32.0(00000)
+1-0:32.36.0(00003)
+0-0:96.13.0()
+1-0:32.7.0(236.0*V)
+1-0:31.7.0(005*A)
+1-0:21.7.0(01.286*kW)
+1-0:22.7.0(00.000*kW)
+0-1:24.1.0(003)
+0-1:96.1.0(4730303933303034333734333338333235)
+0-1:24.2.1(260103201000W)(00272.130*m3)
+!B5E2
+/CTA5ZIV-METER
+
+1-3:0.2.8(50)
+0-0::1.8.2(001130.208*kWh)
+1-0:2.8.1(000080.565*kWh)
+1-0:2.8.2(000205.556*kWh)
+0-0:96.14.0(0001)
+1-0:1.7.0(01.335*kW)
+1-0:2.7.0(00.000*kW)
+0-0:96.7.21(00015)
+0-0:96.7.9(00004)
+1-0:99.97.0(0)(0-0:96.7.19)
+1-0:32.32.0(00000)
+1-0:32.36.0(00003)
+0-0:96.13.0()
+1-0:32.7.0(236.0*V)
+1-0:31.7.0(005*A)
+1-0:21.7.0(01.335*kW)
+1-0:22.7.0(00.000*kW)
+0-1:24.1.0(003)
+0-1:96.1.0(4730303933303034333734333338333235)
+0-1:24.2.1(260103201000W)(00272.130*m3)
+!6D5D
+/CTA5ZIV-METER
+
+1-3:0.2.8(50)
+0-0:1.0.0(260103201104W)
+
+1-0:2.8.1(000080.565*kWh)
+1-0:2.8.2(000205.556*kWh)
+0-0:96.14.0(0001)
+1-0:1.7.0(01.276*kW)
+1-0:2.7.0(00.000*kW)
+0-0:96.7.21(00015)
+0-0:96.7.9(00004)
+1-0:99.97.0(0)(0-0:96.7.19)
+1-0:32.32.0(00000)
+1-0:32.36.0(00003)
+0-0:96.13.0()
+1-0:32.7.0(236.0*V)
+1-0:31.7.0(005*A)
+1-0:21.7.0(01.276*kW)
+1-0:22.7.0(00.000*kW)
+0-1:24.1.0(003)
+0-1:96.1.0(4730303933303034333734333338333235)
+0-1:24.2.1(260103201000W)(00272.130*m3)
+!FDB1
+3233)
+1-0:1.8.1(001402.673*kWh)
+1-0:1.8.2(001130.208*kWh)
+1-0:2.8.1(000080.565*kWh)
+1-0:2.8.2(000205.556*kWh)
+0-0:96.14.0(0001)
+1-0:1.7.0(01.344*kW)
+1-0:2.7.0(00.000*kW)
+0-0:96.7.21(00015)
+0-0:96.7.9(00004)
+1-0:99.97.0(0)(0-0:96.7.19)
+1-0:32.32.0(00000)
+1-0:32.36.0(00003)
+0-0:96.13.0()
+1-0:32.7.0(236.0*V)
+1-0:31.7.0(006*A)
+1-0:21.7.0(01.344*kW)
+1-0:22.7.0(00.000*kW)
+0-1:24.1.0(003)
+0-1:96.1.0(4730303933303034333734333338333235)
+0-1:24.2.1(260103201000W)(00272.130*m3)
+!E9B8
+/CTA5ZIV-METER
+
+1-3:0.2.3*kWh)
+1-0:1.8.2(001130.208*kWh)
+1-0:2.8.1(000080.565*kWh)
+1-0:2.8.2(000205.556*kWh)
+0-0:96.14.0(0001)
+1-0:1.7.0(01.319*kW)
+1-0:2.7.0(00.000*kW)
+0-0:96.7.21(00015)
+0-0:96.7.9(00004)
+1-0:99.97.0(0)(0-0:96.7.19)
+1-0:32.32.0(00000)
+1-0:32.36.0(00003)
+0-0:96.13.0()
+1-0:32.7.0(236.0*V)
+1-0:31.7.0(005*A)
+1-0:21.7.0(01.319*kW)
+1-0:22.7.0(00.000*kW)
+0-1:24.1.0(003)
+0-1:96.1.0(4730303933303034333734333338333235)
+0-1:24.2.1(260103201000W)(00272.130*m3)
+!F0CD
+/CTA5ZIV-METER
+
+1-3:0.2.8(50)
+0-0:1.0.0(2001130.208*kWh)
+1-0:2.8.1(000080.565*kWh)
+1-0:2.8.2(000205.556*kWh)
+0-0:96.14.0(0001)
+1-0:1.7.0(01.336*kW)
+1-0:2.7.0(00.000*kW)
+0-0:96.7.21(00015)
+0-0:96.7.9(00004)
+1-0:99.97.0(0)(0-0:96.7.19)
+1-0:32.32.0(00000)
+1-0:32.36.0(00003)
+0-0:96.13.0()
+1-0:32.7.0(236.0*V)
+1-0:31.7.0(005*A)
+1-0:21.7.0(01.336*kW)
+1-0:22.7.0(00.000*kW)
+0-1:24.1.0(003)
+0-1:96.1.0(4730303933303034333734333338333235)
+0-1:24.2.1(260103201000W)(00272.130*m3)
+!CCE2
+/CTA5ZIV-METER
+
+1-3:0.2.8(50)
+0-0:1.0.0(260103201108W)
+0-0:96.1.1(4530303839303031303230373731393233)
+1-0:1.8.1(001402.674*kWh)
+1-0:1.8.2(001130.208*kWh)
+1-0:2.8.1(000080.565*kWh)
+1-0:2.8.2(000205.556*kWh)
+0-0:96.14.0(0001)
+1-0:1.7.0(01.319*kW)
+1-0:2.7.0(00.000*kW)
+0-0:96.7.21(00015)
+0-0:96.7.9(00004)
+1-0:99.97.0(0)(0-0:96.7.19)
+1-0:32.32.0(00000)
+1-0:32.36.0(00003)
+0-0:96.13.0()
+1-0:32.7.0(236.0*V)
+1-0:31.7.0(005*A)
+1-0:21.7.0(01.319*kW)
+1-0:22.7.0(00.000*kW)
+0-1:24.1.0-1:96.1.0(4730303933303034333734333338333235)
+0-1:24.2.1(260103201000W)(00272.130*m3)
+!B4B3

frankenergy.py

@@ -0,0 +1,46 @@
+import requests
+from datetime import datetime, timedelta
+import csv
+
+def FrankEnergy():
+    now = datetime.now()
+
+    yesterday = datetime.now() + timedelta(days=-1)
+    startdate = yesterday.strftime("%Y-%m-%d")
+    enddate = now.strftime("%Y-%m-%d")
+
+    if int(now.strftime("%H")) > 15:
+        tomorrow = datetime.now() + timedelta(days=2)
+        startdate = now.strftime("%Y-%m-%d")
+        enddate = tomorrow.strftime("%Y-%m-%d")
+
+    headers = { "content-type":"application/json" }
+
+    query = f"""query MarketPrices {{
+        marketPricesGas(startDate: "{startdate}", endDate: "{enddate}") {{
+        from
+        till
+        marketPrice
+        priceIncludingMarkup
+    }}
+    }}"""
+
+    response = requests.post('https://graphcdn.frankenergie.nl', json={'query': query}, headers=headers)
+    data = response.json()
+
+    frank_gas_file = "frank_gas.csv"
+
+    frank_headers = ['till', 'from', 'marketPrice', 'priceIncludingMarkup']
+
+    with open(frank_gas_file, 'w') as csvfile:
+        writer = csv.DictWriter(csvfile, fieldnames = frank_headers)
+        writer.writeheader()
+        writer.writerows(data['data']['marketPricesGas'])
+
+    for gas in data['data']['marketPricesGas']:
+        print(gas['till'], gas['from'],gas['marketPrice'], gas['priceIncludingMarkup'])
+
+
+if __name__ == "__main__":
+    FrankEnergy()
+

pids/smartmeter.pid

@@ -0,0 +1 @@
+19407

report_mailer.rb

@@ -0,0 +1,24 @@
+require "rubygems"
+require "bundler/setup"
+require "active_record"
+require "state_pattern"
+
+project_root = File.dirname(File.absolute_path(__FILE__))
+Dir.glob(project_root + "/app/models/*.rb").each{|f| require f}
+Dir.glob(project_root + "/app/helpers/SearchingForSyncState.rb").each{|f| require f}
+Dir.glob(project_root + "/app/helpers/*.rb").each{|f| require f}
+  
+connection_details = YAML::load(File.open('config/database.yml'))
+ActiveRecord::Base.establish_connection(connection_details)
+
+if __FILE__ == $0
+  yesterday = Date.today.advance(days: -1)
+  ReadingsMailer.deliver(yesterday)
+
+  # are we at the start of a month?
+  if (Date.today.beginning_of_month == Date.today)
+    # then also send report for previous month (and year)
+    ReadingsMailer.deliver_for_month(yesterday.month, yesterday.year)
+  end  
+end
+#p sync

rstudio/.config/rstudio/rstudio-prefs.json

@@ -0,0 +1 @@
+{}

rstudio/.local/share/rstudio/addin_registry

@@ -0,0 +1,83 @@
+{
+    "clipr::clipr_output": {
+        "name": "Output to clipboard",
+        "package": "clipr",
+        "title": "",
+        "description": "Copies the console output of a selected expression to the system clipboard",
+        "interactive": false,
+        "binding": "clipr_output",
+        "ordinal": 2
+    },
+    "clipr::clipr_result": {
+        "name": "Value to clipboard",
+        "package": "clipr",
+        "title": "",
+        "description": "Copies the results of a selected expression to the system clipboard",
+        "interactive": false,
+        "binding": "clipr_result",
+        "ordinal": 1
+    },
+    "devtools::document": {
+        "name": "Document a package",
+        "package": "devtools",
+        "title": "",
+        "description": "A wrapper for `roxygen`'s `roxygen2::roxygenize()`",
+        "interactive": true,
+        "binding": "document",
+        "ordinal": 6
+    },
+    "devtools::run_examples": {
+        "name": "Run examples",
+        "package": "devtools",
+        "title": "",
+        "description": "Runs R code in examples using `devtools::run_examples()`",
+        "interactive": true,
+        "binding": "run_examples",
+        "ordinal": 7
+    },
+    "devtools::test_active_file": {
+        "name": "Run a test file",
+        "package": "devtools",
+        "title": "",
+        "description": "Run the current test file, using `devtools::test_active_file()`.",
+        "interactive": true,
+        "binding": "test_active_file",
+        "ordinal": 3
+    },
+    "devtools::test_coverage": {
+        "name": "Report test coverage for a package",
+        "package": "devtools",
+        "title": "",
+        "description": "Calculate and report the test coverage for the current package, using `devtools::test_coverage()`.",
+        "interactive": true,
+        "binding": "test_coverage",
+        "ordinal": 5
+    },
+    "devtools::test_coverage_active_file": {
+        "name": "Report test coverage for a file",
+        "package": "devtools",
+        "title": "",
+        "description": "Calculate and report test coverage for the current test file, using `devtools::test_coverage_active_file()`.",
+        "interactive": true,
+        "binding": "test_coverage_active_file",
+        "ordinal": 4
+    },
+    "reprex::reprex_addin": {
+        "name": "Render reprex...",
+        "package": "reprex",
+        "title": "",
+        "description": "Run `reprex::reprex()` to prepare a reproducible example for sharing.",
+        "interactive": true,
+        "binding": "reprex_addin",
+        "ordinal": 8
+    },
+    "reprex::reprex_selection": {
+        "name": "Reprex selection",
+        "package": "reprex",
+        "title": "",
+        "description": "Prepare reprex from current selection",
+        "interactive": false,
+        "binding": "reprex_selection",
+        "ordinal": 9
+    }
+}

rstudio/.local/share/rstudio/client-state/console.temporary

@@ -0,0 +1,3 @@
+{
+    "input": ""
+}

rstudio/.local/share/rstudio/client-state/environment-grid.persistent

@@ -0,0 +1,3 @@
+{
+    "objectDisplayType": 0
+}

rstudio/.local/share/rstudio/client-state/environment-panel.temporary

@@ -0,0 +1,8 @@
+{
+    "environmentPanelSettings": {
+        "scroll_position": 0,
+        "expanded_objects": [],
+        "sort_column": 0,
+        "ascending_sort": true
+    }
+}

rstudio/.local/share/rstudio/client-state/source-column-manager.persistent

@@ -0,0 +1,8 @@
+{
+    "column-info": {
+        "names": [
+            "Source"
+        ],
+        "activeColumn": "Source"
+    }
+}

rstudio/.local/share/rstudio/client-state/workbenchp.persistent

@@ -0,0 +1,9 @@
+{
+    "rightpanesize": {
+        "panelwidth": 1485,
+        "windowwidth": 1501,
+        "splitterpos": [
+            675
+        ]
+    }
+}

rstudio/.local/share/rstudio/history_database

@@ -0,0 +1,9 @@
+1641994941352:library(DBI)
+1641994972245:install.packages("RMariaDB")
+1641995012317:library(DBI)
+1641996642217:library(DBI)
+1641996662061:con <- dbConnect(RMariaDB::MariaDB())
+1641999172022:library(DBI)
+1641999197937:con <- dbConnect(RMariaDB::MariaDB())
+1642003072441:con <- dbConnect(RMariaDB::MariaDB(),host="db")
+1642003092194:con <- dbConnect(RMariaDB::MariaDB(),user="root",host="db")

rstudio/.local/share/rstudio/pcs/files-pane.pper

@@ -0,0 +1,9 @@
+{
+    "sortOrder": [
+        {
+            "columnIndex": 2,
+            "ascending": true
+        }
+    ],
+    "path": "~"
+}

rstudio/.local/share/rstudio/pcs/packages-pane.pper

@@ -0,0 +1,7 @@
+{
+    "installOptions": {
+        "installFromRepository": true,
+        "libraryPath": "/usr/local/lib/R/site-library",
+        "installDependencies": true
+    }
+}

rstudio/.local/share/rstudio/pcs/source-pane.pper

@@ -0,0 +1,3 @@
+{
+    "activeTab": -1
+}

rstudio/.local/share/rstudio/pcs/windowlayoutstate.pper

@@ -0,0 +1,14 @@
+{
+    "left": {
+        "splitterpos": 319,
+        "topwindowstate": "HIDE",
+        "panelheight": 724,
+        "windowheight": 798
+    },
+    "right": {
+        "splitterpos": 478,
+        "topwindowstate": "NORMAL",
+        "panelheight": 724,
+        "windowheight": 798
+    }
+}

rstudio/.local/share/rstudio/pcs/workbench-pane.pper

@@ -0,0 +1,5 @@
+{
+    "TabSet1": 0,
+    "TabSet2": 0,
+    "TabZoom": {}
+}

rstudio/.local/share/rstudio/persistent-state

@@ -0,0 +1,2 @@
+activeClientUrl="http://localhost:8787/"
+portToken="97cf3656f4f2"

rstudio/.local/share/rstudio/rstudio-server.json

@@ -0,0 +1,3 @@
+{
+    "context_id": "6A00CEBA"
+}

rstudio/.local/share/rstudio/sessions/active/session-c5601715/properites/executing

@@ -0,0 +1 @@
+0

rstudio/.local/share/rstudio/sessions/active/session-c5601715/properites/initial

@@ -0,0 +1 @@
+0

rstudio/.local/share/rstudio/sessions/active/session-c5601715/properites/last-used

@@ -0,0 +1 @@
+1641999151566.000000

rstudio/.local/share/rstudio/sessions/active/session-c5601715/properites/project

@@ -0,0 +1 @@
+none

rstudio/.local/share/rstudio/sessions/active/session-c5601715/properites/r-version

@@ -0,0 +1 @@
+4.1.2

rstudio/.local/share/rstudio/sessions/active/session-c5601715/properites/r-version-home

@@ -0,0 +1 @@
+/usr/local/lib/R

rstudio/.local/share/rstudio/sessions/active/session-c5601715/properites/running

@@ -0,0 +1 @@
+1

rstudio/.local/share/rstudio/sessions/active/session-c5601715/properites/save_prompt_required

@@ -0,0 +1 @@
+1

rstudio/.local/share/rstudio/sessions/active/session-c5601715/properites/working-dir

@@ -0,0 +1 @@
+~

rstudio/.local/share/rstudio/sessions/active/session-c5601715/session-persistent-state

@@ -0,0 +1,2 @@
+abend="1"
+active-client-id="cfb30428-aa38-4807-8006-002265146fba"

rstudio/.local/share/rstudio/sessions/active/session-c5601715/suspended-session-data/environment_vars

@@ -0,0 +1,64 @@
+CLICOLOR_FORCE="1"
+CRAN="https://packagemanager.rstudio.com/cran/__linux__/focal/latest"
+CWD="/"
+DEFAULT_USER="rstudio"
+DISPLAY=":0"
+EDITOR="vi"
+GIT_ASKPASS="rpostback-askpass"
+HOME="/home/rstudio"
+HOSTNAME="c54de4659c49"
+LANG="en_US.UTF-8"
+LD_LIBRARY_PATH="/usr/local/lib/R/lib:/lib:/usr/local/lib:/usr/lib/x86_64-linux-gnu:/usr/lib/jvm/java-11-openjdk-amd64/lib/server"
+LN_S="ln -s"
+LOGNAME="rstudio"
+MAKE="make"
+MPLENGINE="tkAgg"
+PAGER="/usr/bin/pager"
+PATH="/usr/lib/rstudio-server/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/lib/rstudio-server/bin/postback"
+RMARKDOWN_MATHJAX_PATH="/usr/lib/rstudio-server/resources/mathjax-27"
+RSTUDIO="1"
+RSTUDIO_CONSOLE_COLOR="256"
+RSTUDIO_CONSOLE_WIDTH="104"
+RSTUDIO_HTTP_REFERER="http://localhost:8787/"
+RSTUDIO_PANDOC="/usr/lib/rstudio-server/bin/pandoc"
+RSTUDIO_PROGRAM_MODE="server"
+RSTUDIO_R_MODULE=""
+RSTUDIO_R_PRELAUNCH_SCRIPT=""
+RSTUDIO_R_REPO=""
+RSTUDIO_R_VERSION_LABEL=""
+RSTUDIO_SESSION_STREAM="rstudio-d"
+RSTUDIO_USER_IDENTITY="rstudio"
+RSTUDIO_USER_IDENTITY_DISPLAY="rstudio"
+RSTUDIO_WINUTILS="bin/winutils"
+RS_RPOSTBACK_PATH="/usr/lib/rstudio-server/bin/rpostback"
+RS_SESSION_TMP_DIR="/var/run/rstudio-server/rstudio-rsession"
+R_BROWSER="xdg-open"
+R_BZIPCMD="/usr/bin/bzip2"
+R_DOC_DIR="/usr/local/lib/R/doc"
+R_GZIPCMD="/usr/bin/gzip"
+R_HOME="/usr/local/lib/R"
+R_INCLUDE_DIR="/usr/local/lib/R/include"
+R_LIBS="/usr/local/lib/R/site-library:/usr/local/lib/R/library"
+R_LIBS_SITE=""
+R_LIBS_USER="~/R/x86_64-pc-linux-gnu-library/4.1"
+R_PAPERSIZE="letter"
+R_PDFVIEWER="/usr/bin/xdg-open"
+R_PLATFORM="x86_64-pc-linux-gnu"
+R_PRINTCMD="/usr/bin/lpr"
+R_RD4PDF="times,inconsolata,hyper"
+R_SESSION_TMPDIR="/tmp/RtmpL2PZYh"
+R_SHARE_DIR="/usr/local/lib/R/share"
+R_STRIP_SHARED_LIB="strip --strip-unneeded"
+R_STRIP_STATIC_LIB="strip --strip-debug"
+R_SYSTEM_ABI="linux,gcc,gxx,gfortran,gfortran"
+R_TEXI2DVICMD="/usr/bin/texi2dvi"
+R_UNZIPCMD="/usr/bin/unzip"
+R_VERSION="4.1.2"
+R_ZIPCMD="/usr/bin/zip"
+S6_VERSION="v2.1.0.2"
+SED="/usr/bin/sed"
+SSH_ASKPASS="rpostback-askpass"
+TAR="/usr/bin/tar"
+TERM="xterm"
+TZ="Etc/UTC"
+USER="rstudio"

rstudio/.local/share/rstudio/sessions/active/session-c5601715/suspended-session-data/rversion

@@ -0,0 +1 @@
+4.1.2

rstudio/.local/share/rstudio/sessions/active/session-c5601715/suspended-session-data/settings

@@ -0,0 +1,2 @@
+packrat_mode_on="0"
+r_profile_on_restore="1"

rstudio/.my.cnf

@@ -0,0 +1,3 @@
+[smartmeter]
+user="root"
+password="rootme"

rstudio/Dockerfile

@@ -0,0 +1,21 @@
+FROM rocker/tidyverse:latest
+
+RUN apt-get update \
+        && apt-get install -y libmariadb-dev \
+		libicu-dev liblzma-dev libpcre3-dev libpng-dev \
+                libv8-dev libbz2-dev libxml2-dev \
+        && apt-get clean \ 
+        && rm -rf /var/lib/apt/lists/ \ 
+        && rm -rf /tmp/downloaded_packages/ /tmp/*.rds
+    
+## install packages from CRAN (and clean up)
+RUN Rscript -e "install.packages(c('tidyverse','purr','psych','lme4','lmerTest','broom','doBy','reshape','emmeans','effects','mlr','randomForest','glmnet','foreign'), repos='https://cran.rstudio.com/')" \
+    && rm -rf /tmp/downloaded_packages/ /tmp/*.rds
+
+RUN mkdir /home/rstudio/smartmeter
+VOLUME /home/rstudio/smartmeter
+
+EXPOSE 8787
+
+CMD ["/init"]
+

rstudio/Energy.Rmd

@@ -0,0 +1,77 @@
+library(dplyr)
+library(dbplyr)
+library(ggplot2)
+library(lubridate)
+library(tidyr)
+
+# Access to our smartmeter DB
+con <- DBI::dbConnect(RMariaDB::MariaDB(), host="db", dbname="smartmeter", user="root", password="rootme")
+
+DBI::dbListTables(con)
+
+# get some records
+res <- DBI::dbSendQuery(con, "SELECT * FROM readings LIMIT 10")
+DBI::dbFetch(res)
+
+# dplyr style of reading data from mysql
+# con %>% tbl("readings") %>% show_query()
+
+# get all records from 2022
+energy <- con %>% tbl("readings") %>% 
+        select(total_m3_gas_consumed, total_kwh_consumed_high, total_kwh_consumed_low, 
+                total_kwh_produced_high, total_kwh_produced_low, high_tarif, created_at) %>% 
+        filter(created_at > "2021-09-01", created_at < "2021-12-31") %>%
+        mutate(date = as.Date(created_at)) %>%
+        collect()
+
+# add hour column
+energy <- energy %>% mutate(hour = format(strptime(created_at,"%Y-%m-%d %H:%M:%S"),'%H'))
+
+# group by hour
+energy_per_hour <- energy %>% 
+                mutate(total_usage_kwh = (total_kwh_consumed_high+total_kwh_consumed_low)) %>%
+                mutate(total_return_kwh = (total_kwh_produced_high+total_kwh_produced_low)) %>%
+                group_by(date,hour) %>% 
+                summarize(max_m3_gas_consumed = max(total_m3_gas_consumed), 
+                            min_m3_gas_consumed = min(total_m3_gas_consumed), 
+                            max_usage_kwh = max(total_usage_kwh), 
+                            min_usage_kwh = min(total_usage_kwh), 
+                            max_return_kwh = max(total_return_kwh), 
+                            min_return_kwh = min(total_return_kwh)) %>%
+                mutate(usage_m3 = max_m3_gas_consumed-min_m3_gas_consumed) %>%
+                mutate(usage_kwh = max_usage_kwh-min_usage_kwh) %>%
+                mutate(return_kwh = max_return_kwh-min_return_kwh) %>%
+                select(-max_m3_gas_consumed, -min_m3_gas_consumed, -max_usage_kwh, -min_usage_kwh, -max_return_kwh, -min_return_kwh )
+
+# mutate(usage_kwh = round(usage_kwh,1), return_kwh = round(return_kwh,1)) %>%
+# and again, group by day
+energy_per_day <- energy_per_hour %>% 
+                  group_by(date) %>% 
+                  summarize(usage_m3=round(sum(usage_m3),2), 
+                            usage_kwh=round(sum(usage_kwh),2),
+                            return_kwh=round(sum(return_kwh),2))
+
+# some plots
+energy_per_hour %>% 
+            mutate(usage_kwh = round(usage_kwh,1), return_kwh = round(return_kwh,1)) %>%
+            ggplot( aes(x=date, fill=hour, y=usage_kwh, text=as.character(date))) +
+            geom_bar(stat="identity") +
+            theme_bw() +  
+            labs(x="Date", y="kwh") 
+
+# daily usage/return of electricity
+energy_per_day %>% pivot_longer(cols = usage_kwh:return_kwh) %>%
+            ggplot( aes(x=date, y=value, fill=name, text=as.character(date))) +
+            geom_bar(position="dodge", stat="identity") +
+            geom_text(aes(label=value), vjust=-0.3, hjust=1.2, size=2.5) +
+            theme_bw() +  
+            labs(x="Date", y="kwh") 
+
+# daily usage/return of gas
+energy_per_day %>% pivot_longer(cols = usage_m3) %>%
+            ggplot( aes(x=date, y=value, fill=name, text=as.character(date))) +
+            geom_bar(position="dodge", stat="identity") +
+            geom_text(aes(label=value), vjust=-0.3, hjust=1.2, size=2.5) +
+            theme_bw() +  
+            labs(x="Date", y="m3") 
+

smartmeter.rb

@@ -0,0 +1,52 @@
+#require "rubygems"
+require "bundler/setup"
+require "active_record"
+require "serialport"
+require "state_pattern"
+
+MAX_BYTES = 100
+
+project_root = File.dirname(File.absolute_path(__FILE__))
+Dir.glob(project_root + "/app/models/*.rb").each{|f| require f}
+Dir.glob(project_root + "/app/helpers/SearchingForSyncState.rb").each{|f| require f}
+Dir.glob(project_root + "/app/helpers/*.rb").each{|f| require f}
+  
+connection_details = YAML::load(File.open('config/database.yml'))
+ActiveRecord::Base.establish_connection(connection_details)
+
+def open_device
+  begin
+    # Open connection to serial port
+    io_device = SerialPort.new("/dev/ttyUSB0", 115200, 8, 1, SerialPort::NONE)
+    # Make reading blocking
+    io_device.read_timeout = 0
+  rescue 
+    p "Serialport Error - reverting to 'example_blurp.txt'"
+    io_device = File.open("example_blurp.txt")
+    # start at random place in the file
+    io_device.seek(rand(1500), IO::SEEK_SET)
+  end 
+  return io_device
+end
+
+def read_from(source)
+  source.read(MAX_BYTES).gsub(/\r/, '') rescue ""
+end
+
+if __FILE__ == $0
+  # open serial port and read first bytes
+  source = open_device
+  buffer = []
+  buffer = read_from(source)
+  
+  #
+  # Process the received lines
+  #
+  sync = Synchronizer.new
+  while (buffer && buffer.length > 0)
+  #  p "BUFFER: #{buffer}."
+    buffer = sync.handle_byte_stream(buffer)
+    buffer = buffer + read_from(source)
+  end
+end
+#p sync

tariff_mailer.rb

@@ -0,0 +1,18 @@
+require "rubygems"
+require "bundler/setup"
+require "active_record"
+require "state_pattern"
+
+project_root = File.dirname(File.absolute_path(__FILE__))
+Dir.glob(project_root + "/app/models/*.rb").each{|f| require f}
+Dir.glob(project_root + "/app/helpers/SearchingForSyncState.rb").each{|f| require f}
+Dir.glob(project_root + "/app/helpers/*.rb").each{|f| require f}
+  
+connection_details = YAML::load(File.open('config/database.yml'))
+ActiveRecord::Base.establish_connection(connection_details)
+
+if __FILE__ == $0
+  # Tomorrows tariffs delivered today
+  TariffsMailer.deliver(Date.today.advance(days: 1))
+end
+#p sync

test-serial.rb

@@ -0,0 +1,39 @@
+require "rubygems"
+require "bundler/setup"
+require "active_record"
+require "serialport"
+require "state_pattern"
+
+
+project_root = File.dirname(File.absolute_path(__FILE__))
+Dir.glob(project_root + "/app/models/*.rb").each{|f| require f}
+Dir.glob(project_root + "/app/helpers/SearchingForSyncState.rb").each{|f| require f}
+Dir.glob(project_root + "/app/helpers/*.rb").each{|f| require f}
+  
+connection_details = YAML::load(File.open('config/database.yml'))
+ActiveRecord::Base.establish_connection(connection_details)
+
+
+if __FILE__ == $0
+
+  #params for serial port
+  port_str = "/dev/ttyUSB0"  #may be different for you
+  baud_rate = 115200
+  data_bits = 8
+  stop_bits = 1
+  parity = SerialPort::NONE
+
+sp = SerialPort.new(port_str, baud_rate, data_bits, stop_bits, parity)
+
+# Make reading blocking
+sp.read_timeout = 0
+
+#just read forever
+while true do
+  printf("%c", sp.getc)
+end
+
+sp.close                       #see note 1
+
+end
+