Durable notes and offline time offsets

gonotego/command_center/commands.py

@@ -10,6 +10,7 @@
 from gonotego.command_center import note_commands  # noqa: F401
 from gonotego.command_center import settings_commands  # noqa: F401
 from gonotego.command_center import system_commands  # noqa: F401
+from gonotego.command_center import time_commands  # noqa: F401
 from gonotego.command_center import twitter_commands  # noqa: F401
 from gonotego.command_center import wifi_commands  # noqa: F401
 from gonotego.command_center import registry

gonotego/command_center/note_commands.py

@@ -4,6 +4,8 @@
 from gonotego.command_center import system_commands
 from gonotego.common import events
 from gonotego.common import interprocess
+from gonotego.common import note_log
+from gonotego.common import time_offsets
 
 
 register_command = registry.register_command
@@ -13,51 +15,49 @@ def get_timestamp():
   return time.time()
 
 
+def put_note_event(note_event):
+  """Logs a note event locally and enqueues it for the uploader."""
+  note_event.offset = time_offsets.get_offset()
+  note_log.log(note_event)
+  interprocess.get_note_events_queue().put(bytes(note_event))
+  interprocess.get_note_events_session_queue().put(bytes(note_event))
+
+
 @register_command('note {}')
 def add_note(text):
-  note_events_queue = interprocess.get_note_events_queue()
-  note_events_session_queue = interprocess.get_note_events_session_queue()
-
   note_event = events.NoteEvent(
       text=text,
       action=events.SUBMIT,
       audio_filepath=None,
       timestamp=get_timestamp())
-  note_events_queue.put(bytes(note_event))
-  note_events_session_queue.put(bytes(note_event))
+  put_note_event(note_event)
 
 
 @register_command('subnote {}')
 def add_indented_note(text):
-  note_events_queue = interprocess.get_note_events_queue()
-  note_events_session_queue = interprocess.get_note_events_session_queue()
-
   # Indent
   note_event = events.NoteEvent(
       text=None,
       action=events.INDENT,
       audio_filepath=None,
       timestamp=get_timestamp())
-  note_events_queue.put(bytes(note_event))
-  note_events_session_queue.put(bytes(note_event))
+  put_note_event(note_event)
 
   # The note
   note_event = events.NoteEvent(
       text=text,
       action=events.SUBMIT,
       audio_filepath=None,
       timestamp=get_timestamp())
-  note_events_queue.put(bytes(note_event))
-  note_events_session_queue.put(bytes(note_event))
+  put_note_event(note_event)
 
   # Dedent
   note_event = events.NoteEvent(
       text=None,
       action=events.UNINDENT,
       audio_filepath=None,
       timestamp=get_timestamp())
-  note_events_queue.put(bytes(note_event))
-  note_events_session_queue.put(bytes(note_event))
+  put_note_event(note_event)
 
 
 @register_command('todo {}')

gonotego/command_center/time_commands.py

@@ -0,0 +1,81 @@
+"""Commands for correcting note timestamps when the system clock is wrong.
+
+Go Note Go has no realtime clock, so after a flight (or a long stretch
+offline) the system clock can be off. These commands manage an
+'alleged time' offset:
+
+  :xtime 3:30pm   -- declare what time it really is. Future notes carry both
+                     the raw clock time and the offset to the alleged time.
+  :xtime          -- say the current alleged time and offset.
+  :xtime clear    -- clear the offset.
+  :rxtime 4       -- retroactively apply the current offset to the last
+                     4 hours of not-yet-uploaded notes.
+  :rxtime         -- same, reaching back to the last boot.
+"""
+
+from datetime import datetime
+import time
+
+from gonotego.command_center import registry
+from gonotego.command_center import system_commands
+from gonotego.common import time_offsets
+
+register_command = registry.register_command
+
+CLEAR_WORDS = ('clear', 'off', 'reset', 'none')
+
+
+@register_command('xtime {}')
+def set_alleged_time(time_str):
+  """Sets the alleged time, recording its offset from the system clock."""
+  time_str = time_str.strip()
+  if time_str.lower() in CLEAR_WORDS:
+    time_offsets.clear_offset()
+    system_commands.say('time offset cleared')
+    return
+  alleged_dt = time_offsets.parse_alleged_time(time_str)
+  if alleged_dt is None:
+    system_commands.say(f'could not parse time {time_str}')
+    return
+  offset = time_offsets.compute_offset(alleged_dt)
+  time_offsets.set_offset(offset)
+  say_alleged_time(prefix=f'offset {time_offsets.describe_offset(offset)}. ')
+
+
+@register_command('xtime')
+def say_alleged_time(prefix=''):
+  """Says the current alleged time and offset."""
+  offset = time_offsets.get_offset()
+  alleged = datetime.fromtimestamp(time.time() + offset)
+  time_str = alleged.strftime('%A %l:%M %p').strip()
+  system_commands.say(f'{prefix}alleged time {time_str}')
+
+
+@register_command('rxtime {}')
+def apply_offset_to_recent_notes(hours_str):
+  """Retroactively applies the offset to the last N hours of pending notes."""
+  hours_str = hours_str.strip()
+  try:
+    hours = float(hours_str)
+  except ValueError:
+    system_commands.say(f'could not parse hours {hours_str}')
+    return
+  cutoff = time.time() - hours * 60 * 60
+  apply_offset_since(cutoff)
+
+
+@register_command('rxtime')
+def apply_offset_to_notes_since_boot():
+  """Retroactively applies the offset to pending notes since the last boot."""
+  cutoff = time_offsets.boot_timestamp()
+  if cutoff is None:
+    system_commands.say('could not determine boot time')
+    return
+  apply_offset_since(cutoff)
+
+
+def apply_offset_since(cutoff):
+  offset = time_offsets.get_offset()
+  updated = time_offsets.apply_offset_retroactively(offset, cutoff)
+  system_commands.say(
+      f'applied offset {time_offsets.describe_offset(offset)} to {updated} pending notes')

gonotego/common/events.py

@@ -46,6 +46,18 @@ class NoteEvent:
   action: Text
   audio_filepath: Text
   timestamp: datetime
+  # Seconds to add to the raw system-clock timestamp to get the alleged time.
+  # Set via ':xtime' when the system clock is known to be wrong (e.g. no
+  # internet while traveling). Notes carry both the raw timestamp and the
+  # offset, so either time can be recovered later.
+  offset: float = 0.0
+
+  @property
+  def effective_timestamp(self):
+    """The timestamp with the alleged-time offset applied."""
+    if self.timestamp is None:
+      return None
+    return self.timestamp + (self.offset or 0.0)
 
   def __bytes__(self):
     return json.dumps(dataclasses.asdict(self)).encode('utf-8')

gonotego/common/internet.py

@@ -5,22 +5,46 @@
 
 Status = status.Status
 
+# Endpoints that are known to return HTTP 204 with an empty body.
+# Captive portals (hotel wifi login pages, etc.) intercept requests and answer
+# with their own page -- typically a 200 or a redirect with a body. Requiring
+# an exact 204-with-no-content response distinguishes real internet access
+# from a captive portal that is merely accepting TCP connections.
+CONNECTIVITY_CHECK_ENDPOINTS = (
+    ('connectivitycheck.gstatic.com', '/generate_204'),
+    ('clients3.google.com', '/generate_204'),
+)
 
-def is_internet_available(url='www.google.com'):
-  """Determines if we are connected to the Internet."""
-  connection = http.client.HTTPConnection(url, timeout=2)
+
+def check_endpoint(host, path, timeout=2):
+  """Checks a single connectivity-check endpoint for a genuine 204 response."""
+  connection = http.client.HTTPConnection(host, timeout=timeout)
   try:
-    connection.request('HEAD', '/')
-    connection.close()
-    return True
-  except:
-    connection.close()
+    connection.request('GET', path)
+    response = connection.getresponse()
+    body = response.read()
+    return response.status == 204 and not body
+  except Exception:
     return False
+  finally:
+    connection.close()
+
+
+def is_internet_available():
+  """Determines if we are connected to the Internet.
+
+  Returns True only if a known endpoint returns its expected no-content
+  response, so a captive portal doesn't count as having internet.
+  """
+  for host, path in CONNECTIVITY_CHECK_ENDPOINTS:
+    if check_endpoint(host, path):
+      return True
+  return False
 
 
-def wait_for_internet(url='www.google.com', on_disconnect=None):
+def wait_for_internet(on_disconnect=None):
   first = True
-  while not is_internet_available(url):
+  while not is_internet_available():
     if first:
       print('No internet connection available. Sleeping.')
       status.set(Status.INTERNET_AVAILABLE, False)

gonotego/common/interprocess.py

@@ -48,6 +48,21 @@ def commit(self, value):
     assert self.index >= 0
     assert value == pop_value
 
+  def update_items(self, update_fn):
+    """Applies update_fn to every queued item, in place.
+
+    update_fn receives the item's bytes and returns replacement bytes, or
+    None to leave the item unchanged. Returns the number of items updated.
+    """
+    updated = 0
+    values = self.r.lrange(self.key, 0, -1)
+    for i, value in enumerate(values):
+      new_value = update_fn(value)
+      if new_value is not None and new_value != value:
+        self.r.lset(self.key, i, new_value)
+        updated += 1
+    return updated
+
   def size(self):
     return self.r.llen(self.key) - self.index
 

gonotego/common/note_log.py

@@ -0,0 +1,101 @@
+"""Durable local log of every note event.
+
+Every note event is appended to a local JSONL file at capture time, before
+any transcription or upload happens. These files are the local source of
+truth if anything downstream goes wrong (captive portal, uploader bug,
+crash): a note that made it into the log is never lost silently.
+
+Log files are only ever deleted by `cleanup`, which (a) never touches files
+younger than MAX_AGE_DAYS (~6 months) and (b) doesn't delete anything at all
+unless the disk is under real pressure.
+"""
+
+from datetime import datetime
+import json
+import os
+import shutil
+import time
+
+NOTE_LOG_DIR = os.path.join('out', 'note_log')
+
+# Local copies are kept for at least ~6 months no matter what.
+MAX_AGE_DAYS = 183
+
+# Old files are only cleaned up when free disk space drops below this.
+MIN_FREE_BYTES = 1 * 1024 * 1024 * 1024  # 1 GB
+
+# Only files Go Note Go itself writes are eligible for cleanup.
+CLEANUP_SUFFIXES = ('.wav', '.txt', '.jsonl')
+
+
+def log(note_event):
+  """Appends a note event to the durable local log. Never raises."""
+  try:
+    os.makedirs(NOTE_LOG_DIR, exist_ok=True)
+    timestamp = note_event.timestamp
+    dt = datetime.fromtimestamp(timestamp) if timestamp else datetime.now()
+    filepath = os.path.join(NOTE_LOG_DIR, dt.strftime('%Y-%m-%d') + '.jsonl')
+    offset = getattr(note_event, 'offset', 0.0) or 0.0
+    effective_dt = datetime.fromtimestamp(timestamp + offset) if timestamp else dt
+    entry = {
+        'text': note_event.text,
+        'action': note_event.action,
+        'audio_filepath': note_event.audio_filepath,
+        'timestamp': timestamp,
+        'time': dt.isoformat(),
+        'offset': offset,
+        'effective_time': effective_dt.isoformat(),
+    }
+    with open(filepath, 'a') as f:
+      f.write(json.dumps(entry) + '\n')
+    return filepath
+  except Exception as e:
+    # Writing to the log must never prevent a note from being captured.
+    print(f'Failed to write note event to note log: {e!r}')
+    return None
+
+
+def cleanup(directory='out', max_age_days=MAX_AGE_DAYS, min_free_bytes=MIN_FREE_BYTES):
+  """Deletes old local note copies, but only under real disk pressure.
+
+  Never deletes anything newer than max_age_days. Never deletes anything at
+  all unless free disk space is below min_free_bytes. Deletes oldest files
+  first and stops as soon as enough space is free.
+
+  Returns the list of deleted filepaths.
+  """
+  try:
+    free = shutil.disk_usage(directory).free
+  except OSError:
+    return []
+  if free >= min_free_bytes:
+    return []
+
+  now = time.time()
+  max_age_seconds = max_age_days * 24 * 60 * 60
+  candidates = []
+  for dirpath, unused_dirnames, filenames in os.walk(directory):
+    for filename in filenames:
+      if not filename.endswith(CLEANUP_SUFFIXES):
+        continue
+      filepath = os.path.join(dirpath, filename)
+      try:
+        mtime = os.path.getmtime(filepath)
+        size = os.path.getsize(filepath)
+      except OSError:
+        continue
+      if now - mtime > max_age_seconds:
+        candidates.append((mtime, filepath, size))
+
+  candidates.sort()
+  deleted = []
+  for unused_mtime, filepath, size in candidates:
+    if free >= min_free_bytes:
+      break
+    try:
+      os.remove(filepath)
+    except OSError:
+      continue
+    free += size
+    deleted.append(filepath)
+  return deleted

gonotego/common/test_events.py

@@ -29,6 +29,25 @@ def test_note_event(self):
     event2 = events.NoteEvent.from_bytes(event_bytes)
     self.assertEqual(event, event2)
 
+  def test_note_event_with_offset(self):
+    event = events.NoteEvent(
+        text='Example note.',
+        action=events.SUBMIT,
+        audio_filepath='/tmp/audio',
+        timestamp=1000.0,
+        offset=3600.0)
+    event_bytes = bytes(event)
+    event2 = events.NoteEvent.from_bytes(event_bytes)
+    self.assertEqual(event, event2)
+    self.assertEqual(event2.effective_timestamp, 4600.0)
+
+  def test_note_event_from_bytes_without_offset(self):
+    # Events serialized before the offset field existed still deserialize.
+    event_bytes = b'{"text": "Old note.", "action": "submit", "audio_filepath": null, "timestamp": 1000.0}'
+    event = events.NoteEvent.from_bytes(event_bytes)
+    self.assertEqual(event.offset, 0.0)
+    self.assertEqual(event.effective_timestamp, 1000.0)
+
   def test_led_event(self):
     event = events.LEDEvent(
         color=[0, 0, 0],