From 493b0a3db231fab07ecb17da38e56f8af87774a5 Mon Sep 17 00:00:00 2001 From: David Bieber Date: Thu, 16 Jul 2026 21:41:10 +0000 Subject: [PATCH 1/2] Never lose a note: portal-proof internet check, durable local note log Hotel captive portals accept TCP connections, so the old HEAD-request internet check reported internet access while uploads were failing, and note events were dropped from the queue as if uploaded. - is_internet_available now requires a genuine 204-with-empty-body from a generate_204 connectivity endpoint (with a fallback endpoint), so a captive portal no longer counts as being online. - Every note event is appended to a durable local JSONL log (out/note_log/YYYY-MM-DD.jsonl) at capture time, before transcription or upload, from all capture paths (keyboard shell, :note/:subnote commands, voice transcription). - Local copies (note log, .wav recordings, .txt transcripts) are kept for at least 6 months no matter what, and are only cleaned up at all under real disk pressure (<1GB free), oldest first, checked daily by the uploader. Nothing is ever deleted because of an upload. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_016ib1Yk692v5GKFz1bg5hTW --- gonotego/command_center/note_commands.py | 26 +++--- gonotego/common/internet.py | 44 +++++++-- gonotego/common/note_log.py | 97 ++++++++++++++++++++ gonotego/common/test_internet.py | 57 ++++++++++++ gonotego/common/test_note_log.py | 112 +++++++++++++++++++++++ gonotego/text/shell.py | 27 +++--- gonotego/transcription/runner.py | 2 + gonotego/uploader/runner.py | 13 +++ 8 files changed, 342 insertions(+), 36 deletions(-) create mode 100644 gonotego/common/note_log.py create mode 100644 gonotego/common/test_internet.py create mode 100644 gonotego/common/test_note_log.py diff --git a/gonotego/command_center/note_commands.py b/gonotego/command_center/note_commands.py index ccf37c39..abc3b22a 100644 --- a/gonotego/command_center/note_commands.py +++ b/gonotego/command_center/note_commands.py @@ -4,6 +4,7 @@ from gonotego.command_center import system_commands from gonotego.common import events from gonotego.common import interprocess +from gonotego.common import note_log register_command = registry.register_command @@ -13,33 +14,32 @@ def get_timestamp(): return time.time() +def put_note_event(note_event): + """Logs a note event locally and enqueues it for the uploader.""" + note_log.log(note_event) + interprocess.get_note_events_queue().put(bytes(note_event)) + interprocess.get_note_events_session_queue().put(bytes(note_event)) + + @register_command('note {}') def add_note(text): - note_events_queue = interprocess.get_note_events_queue() - note_events_session_queue = interprocess.get_note_events_session_queue() - note_event = events.NoteEvent( text=text, action=events.SUBMIT, audio_filepath=None, timestamp=get_timestamp()) - note_events_queue.put(bytes(note_event)) - note_events_session_queue.put(bytes(note_event)) + put_note_event(note_event) @register_command('subnote {}') def add_indented_note(text): - note_events_queue = interprocess.get_note_events_queue() - note_events_session_queue = interprocess.get_note_events_session_queue() - # Indent note_event = events.NoteEvent( text=None, action=events.INDENT, audio_filepath=None, timestamp=get_timestamp()) - note_events_queue.put(bytes(note_event)) - note_events_session_queue.put(bytes(note_event)) + put_note_event(note_event) # The note note_event = events.NoteEvent( @@ -47,8 +47,7 @@ def add_indented_note(text): action=events.SUBMIT, audio_filepath=None, timestamp=get_timestamp()) - note_events_queue.put(bytes(note_event)) - note_events_session_queue.put(bytes(note_event)) + put_note_event(note_event) # Dedent note_event = events.NoteEvent( @@ -56,8 +55,7 @@ def add_indented_note(text): action=events.UNINDENT, audio_filepath=None, timestamp=get_timestamp()) - note_events_queue.put(bytes(note_event)) - note_events_session_queue.put(bytes(note_event)) + put_note_event(note_event) @register_command('todo {}') diff --git a/gonotego/common/internet.py b/gonotego/common/internet.py index 82c550df..7b1385c1 100644 --- a/gonotego/common/internet.py +++ b/gonotego/common/internet.py @@ -5,22 +5,46 @@ Status = status.Status +# Endpoints that are known to return HTTP 204 with an empty body. +# Captive portals (hotel wifi login pages, etc.) intercept requests and answer +# with their own page -- typically a 200 or a redirect with a body. Requiring +# an exact 204-with-no-content response distinguishes real internet access +# from a captive portal that is merely accepting TCP connections. +CONNECTIVITY_CHECK_ENDPOINTS = ( + ('connectivitycheck.gstatic.com', '/generate_204'), + ('clients3.google.com', '/generate_204'), +) -def is_internet_available(url='www.google.com'): - """Determines if we are connected to the Internet.""" - connection = http.client.HTTPConnection(url, timeout=2) + +def check_endpoint(host, path, timeout=2): + """Checks a single connectivity-check endpoint for a genuine 204 response.""" + connection = http.client.HTTPConnection(host, timeout=timeout) try: - connection.request('HEAD', '/') - connection.close() - return True - except: - connection.close() + connection.request('GET', path) + response = connection.getresponse() + body = response.read() + return response.status == 204 and not body + except Exception: return False + finally: + connection.close() + + +def is_internet_available(): + """Determines if we are connected to the Internet. + + Returns True only if a known endpoint returns its expected no-content + response, so a captive portal doesn't count as having internet. + """ + for host, path in CONNECTIVITY_CHECK_ENDPOINTS: + if check_endpoint(host, path): + return True + return False -def wait_for_internet(url='www.google.com', on_disconnect=None): +def wait_for_internet(on_disconnect=None): first = True - while not is_internet_available(url): + while not is_internet_available(): if first: print('No internet connection available. Sleeping.') status.set(Status.INTERNET_AVAILABLE, False) diff --git a/gonotego/common/note_log.py b/gonotego/common/note_log.py new file mode 100644 index 00000000..08e51867 --- /dev/null +++ b/gonotego/common/note_log.py @@ -0,0 +1,97 @@ +"""Durable local log of every note event. + +Every note event is appended to a local JSONL file at capture time, before +any transcription or upload happens. These files are the local source of +truth if anything downstream goes wrong (captive portal, uploader bug, +crash): a note that made it into the log is never lost silently. + +Log files are only ever deleted by `cleanup`, which (a) never touches files +younger than MAX_AGE_DAYS (~6 months) and (b) doesn't delete anything at all +unless the disk is under real pressure. +""" + +from datetime import datetime +import json +import os +import shutil +import time + +NOTE_LOG_DIR = os.path.join('out', 'note_log') + +# Local copies are kept for at least ~6 months no matter what. +MAX_AGE_DAYS = 183 + +# Old files are only cleaned up when free disk space drops below this. +MIN_FREE_BYTES = 1 * 1024 * 1024 * 1024 # 1 GB + +# Only files Go Note Go itself writes are eligible for cleanup. +CLEANUP_SUFFIXES = ('.wav', '.txt', '.jsonl') + + +def log(note_event): + """Appends a note event to the durable local log. Never raises.""" + try: + os.makedirs(NOTE_LOG_DIR, exist_ok=True) + timestamp = note_event.timestamp + dt = datetime.fromtimestamp(timestamp) if timestamp else datetime.now() + filepath = os.path.join(NOTE_LOG_DIR, dt.strftime('%Y-%m-%d') + '.jsonl') + entry = { + 'text': note_event.text, + 'action': note_event.action, + 'audio_filepath': note_event.audio_filepath, + 'timestamp': timestamp, + 'time': dt.isoformat(), + } + with open(filepath, 'a') as f: + f.write(json.dumps(entry) + '\n') + return filepath + except Exception as e: + # Writing to the log must never prevent a note from being captured. + print(f'Failed to write note event to note log: {e!r}') + return None + + +def cleanup(directory='out', max_age_days=MAX_AGE_DAYS, min_free_bytes=MIN_FREE_BYTES): + """Deletes old local note copies, but only under real disk pressure. + + Never deletes anything newer than max_age_days. Never deletes anything at + all unless free disk space is below min_free_bytes. Deletes oldest files + first and stops as soon as enough space is free. + + Returns the list of deleted filepaths. + """ + try: + free = shutil.disk_usage(directory).free + except OSError: + return [] + if free >= min_free_bytes: + return [] + + now = time.time() + max_age_seconds = max_age_days * 24 * 60 * 60 + candidates = [] + for dirpath, unused_dirnames, filenames in os.walk(directory): + for filename in filenames: + if not filename.endswith(CLEANUP_SUFFIXES): + continue + filepath = os.path.join(dirpath, filename) + try: + mtime = os.path.getmtime(filepath) + size = os.path.getsize(filepath) + except OSError: + continue + if now - mtime > max_age_seconds: + candidates.append((mtime, filepath, size)) + + candidates.sort() + deleted = [] + for unused_mtime, filepath, size in candidates: + if free >= min_free_bytes: + break + try: + os.remove(filepath) + except OSError: + continue + free += size + deleted.append(filepath) + return deleted diff --git a/gonotego/common/test_internet.py b/gonotego/common/test_internet.py new file mode 100644 index 00000000..ec8e48f0 --- /dev/null +++ b/gonotego/common/test_internet.py @@ -0,0 +1,57 @@ +import unittest +from unittest import mock + +from gonotego.common import internet + + +def make_connection(status=204, body=b''): + """Creates a mock http.client.HTTPConnection whose response is fixed.""" + connection = mock.MagicMock() + response = mock.MagicMock() + response.status = status + response.read.return_value = body + connection.getresponse.return_value = response + return connection + + +class InternetTest(unittest.TestCase): + + def test_genuine_204_counts_as_internet(self): + connection = make_connection(status=204, body=b'') + with mock.patch.object(internet.http.client, 'HTTPConnection', return_value=connection): + self.assertTrue(internet.is_internet_available()) + + def test_captive_portal_login_page_is_not_internet(self): + # A captive portal accepts the connection but answers with its own page. + connection = make_connection(status=200, body=b'Hotel wifi login') + with mock.patch.object(internet.http.client, 'HTTPConnection', return_value=connection): + self.assertFalse(internet.is_internet_available()) + + def test_captive_portal_redirect_is_not_internet(self): + connection = make_connection(status=302, body=b'') + with mock.patch.object(internet.http.client, 'HTTPConnection', return_value=connection): + self.assertFalse(internet.is_internet_available()) + + def test_204_with_unexpected_body_is_not_internet(self): + connection = make_connection(status=204, body=b'unexpected') + with mock.patch.object(internet.http.client, 'HTTPConnection', return_value=connection): + self.assertFalse(internet.is_internet_available()) + + def test_connection_error_is_not_internet(self): + connection = mock.MagicMock() + connection.request.side_effect = OSError('no route to host') + with mock.patch.object(internet.http.client, 'HTTPConnection', return_value=connection): + self.assertFalse(internet.is_internet_available()) + + def test_second_endpoint_can_confirm_internet(self): + # If the first endpoint is unreachable but the second returns a genuine + # 204, we are online. + bad = mock.MagicMock() + bad.request.side_effect = OSError('no route to host') + good = make_connection(status=204, body=b'') + with mock.patch.object(internet.http.client, 'HTTPConnection', side_effect=[bad, good]): + self.assertTrue(internet.is_internet_available()) + + +if __name__ == '__main__': + unittest.main() diff --git a/gonotego/common/test_note_log.py b/gonotego/common/test_note_log.py new file mode 100644 index 00000000..b5a07477 --- /dev/null +++ b/gonotego/common/test_note_log.py @@ -0,0 +1,112 @@ +import json +import os +import tempfile +import time +import unittest +from unittest import mock + +from gonotego.common import events +from gonotego.common import note_log + + +class NoteLogTest(unittest.TestCase): + + def setUp(self): + self.tempdir = tempfile.TemporaryDirectory() + self.addCleanup(self.tempdir.cleanup) + + def make_note_event(self, text='Example note.', timestamp=None): + return events.NoteEvent( + text=text, + action=events.SUBMIT, + audio_filepath=None, + timestamp=timestamp if timestamp is not None else time.time()) + + def test_log_writes_note_event(self): + log_dir = os.path.join(self.tempdir.name, 'note_log') + note_event = self.make_note_event(text='A note to keep.') + with mock.patch.object(note_log, 'NOTE_LOG_DIR', log_dir): + filepath = note_log.log(note_event) + self.assertIsNotNone(filepath) + with open(filepath) as f: + entry = json.loads(f.read()) + self.assertEqual(entry['text'], 'A note to keep.') + self.assertEqual(entry['action'], events.SUBMIT) + self.assertEqual(entry['timestamp'], note_event.timestamp) + + def test_log_appends_multiple_events(self): + log_dir = os.path.join(self.tempdir.name, 'note_log') + timestamp = time.time() + with mock.patch.object(note_log, 'NOTE_LOG_DIR', log_dir): + filepath = note_log.log(self.make_note_event(text='one', timestamp=timestamp)) + note_log.log(self.make_note_event(text='two', timestamp=timestamp)) + with open(filepath) as f: + lines = f.read().splitlines() + self.assertEqual([json.loads(line)['text'] for line in lines], ['one', 'two']) + + def test_log_never_raises(self): + # Even with an unwritable log directory, capturing a note must not raise. + log_dir = os.path.join(self.tempdir.name, 'not-a-dir') + with open(log_dir, 'w') as f: + f.write('a file where the directory should be') + with mock.patch.object(note_log, 'NOTE_LOG_DIR', log_dir): + self.assertIsNone(note_log.log(self.make_note_event())) + + def write_file(self, name, age_days, content=b'x' * 1024): + filepath = os.path.join(self.tempdir.name, name) + with open(filepath, 'wb') as f: + f.write(content) + old_time = time.time() - age_days * 24 * 60 * 60 + os.utime(filepath, (old_time, old_time)) + return filepath + + def test_cleanup_is_noop_without_disk_pressure(self): + self.write_file('ancient.wav', age_days=400) + usage = mock.MagicMock(free=note_log.MIN_FREE_BYTES * 10) + with mock.patch.object(note_log.shutil, 'disk_usage', return_value=usage): + deleted = note_log.cleanup(directory=self.tempdir.name) + self.assertEqual(deleted, []) + + def test_cleanup_never_deletes_recent_files(self): + recent = self.write_file('recent.wav', age_days=30) + five_months = self.write_file('five-months.jsonl', age_days=150) + usage = mock.MagicMock(free=0) + with mock.patch.object(note_log.shutil, 'disk_usage', return_value=usage): + deleted = note_log.cleanup(directory=self.tempdir.name) + self.assertEqual(deleted, []) + self.assertTrue(os.path.exists(recent)) + self.assertTrue(os.path.exists(five_months)) + + def test_cleanup_deletes_old_files_under_disk_pressure(self): + old_wav = self.write_file('old.wav', age_days=400) + old_log = self.write_file('old.jsonl', age_days=250) + recent = self.write_file('recent.wav', age_days=10) + usage = mock.MagicMock(free=0) + with mock.patch.object(note_log.shutil, 'disk_usage', return_value=usage): + deleted = note_log.cleanup(directory=self.tempdir.name) + self.assertEqual(sorted(deleted), sorted([old_wav, old_log])) + self.assertFalse(os.path.exists(old_wav)) + self.assertFalse(os.path.exists(old_log)) + self.assertTrue(os.path.exists(recent)) + + def test_cleanup_stops_once_enough_space_is_free(self): + # Oldest file is deleted first; once free space recovers, deletion stops. + oldest = self.write_file('oldest.wav', age_days=500, content=b'x' * 4096) + newer = self.write_file('newer.wav', age_days=300, content=b'x' * 4096) + usage = mock.MagicMock(free=0) + with mock.patch.object(note_log.shutil, 'disk_usage', return_value=usage): + deleted = note_log.cleanup(directory=self.tempdir.name, min_free_bytes=1024) + self.assertEqual(deleted, [oldest]) + self.assertTrue(os.path.exists(newer)) + + def test_cleanup_ignores_foreign_file_types(self): + precious = self.write_file('precious.db', age_days=500) + usage = mock.MagicMock(free=0) + with mock.patch.object(note_log.shutil, 'disk_usage', return_value=usage): + deleted = note_log.cleanup(directory=self.tempdir.name) + self.assertEqual(deleted, []) + self.assertTrue(os.path.exists(precious)) + + +if __name__ == '__main__': + unittest.main() diff --git a/gonotego/text/shell.py b/gonotego/text/shell.py index decd39ed..b220aad9 100644 --- a/gonotego/text/shell.py +++ b/gonotego/text/shell.py @@ -8,6 +8,7 @@ from gonotego.common import events from gonotego.common import interprocess +from gonotego.common import note_log from gonotego.common import status from gonotego.settings import settings @@ -72,6 +73,13 @@ def __init__(self): def start(self): keyboard.on_press(self.on_press) + def put_note_event(self, note_event, session=True): + """Logs a note event locally and enqueues it for the uploader.""" + note_log.log(note_event) + self.note_events_queue.put(bytes(note_event)) + if session: + self.note_events_session_queue.put(bytes(note_event)) + def on_press(self, event): self.last_press = time.time() status.set(Status.TEXT_LAST_KEYPRESS, self.last_press) @@ -91,8 +99,7 @@ def on_press(self, event): action=events.UNINDENT, audio_filepath=None, timestamp=get_timestamp()) - self.note_events_queue.put(bytes(note_event)) - self.note_events_session_queue.put(bytes(note_event)) + self.put_note_event(note_event) else: # Tab note_event = events.NoteEvent( @@ -100,8 +107,7 @@ def on_press(self, event): action=events.INDENT, audio_filepath=None, timestamp=get_timestamp()) - self.note_events_queue.put(bytes(note_event)) - self.note_events_session_queue.put(bytes(note_event)) + self.put_note_event(note_event) elif event.name == 'delete' or event.name == 'backspace': if self.text == '': note_event = events.NoteEvent( @@ -109,8 +115,7 @@ def on_press(self, event): action=events.CLEAR_EMPTY, audio_filepath=None, timestamp=get_timestamp()) - self.note_events_queue.put(bytes(note_event)) - self.note_events_session_queue.put(bytes(note_event)) + self.put_note_event(note_event) self.text = self.text[:-1] if is_shift_pressed(): self.text = '' @@ -126,7 +131,7 @@ def on_press(self, event): action=events.END_SESSION, audio_filepath=None, timestamp=get_timestamp()) - self.note_events_queue.put(bytes(note_event)) + self.put_note_event(note_event, session=False) self.note_events_session_queue.clear() # Write both a text event (for the command center) # and a note event (for the uploader). @@ -136,8 +141,7 @@ def on_press(self, event): action=events.ENTER_EMPTY, audio_filepath=None, timestamp=get_timestamp()) - self.note_events_queue.put(bytes(note_event)) - self.note_events_session_queue.put(bytes(note_event)) + self.put_note_event(note_event) elif self.text.strip().startswith('::'): self.text = self.text.strip()[1:] self.submit_note() @@ -164,8 +168,7 @@ def submit_note(self): action=events.SUBMIT, audio_filepath=None, timestamp=get_timestamp()) - self.note_events_queue.put(bytes(note_event)) - self.note_events_session_queue.put(bytes(note_event)) + self.put_note_event(note_event) # Reset the text buffer. self.text = '' @@ -177,7 +180,7 @@ def handle_inactivity(self): action=events.END_SESSION, audio_filepath=None, timestamp=get_timestamp()) - self.note_events_queue.put(bytes(note_event)) + self.put_note_event(note_event, session=False) self.note_events_session_queue.clear() def wait(self): diff --git a/gonotego/transcription/runner.py b/gonotego/transcription/runner.py index 28d20289..ace7c102 100644 --- a/gonotego/transcription/runner.py +++ b/gonotego/transcription/runner.py @@ -4,6 +4,7 @@ from gonotego.common import events from gonotego.common import internet from gonotego.common import interprocess +from gonotego.common import note_log from gonotego.common import status from gonotego.transcription import transcriber @@ -46,6 +47,7 @@ def main(): audio_filepath=event.filepath, timestamp=time.time(), ) + note_log.log(note_event) note_events_queue.put(bytes(note_event)) note_events_session_queue.put(bytes(note_event)) diff --git a/gonotego/uploader/runner.py b/gonotego/uploader/runner.py index fbaf9967..c34dc390 100644 --- a/gonotego/uploader/runner.py +++ b/gonotego/uploader/runner.py @@ -4,6 +4,7 @@ from gonotego.common import events from gonotego.common import internet from gonotego.common import interprocess +from gonotego.common import note_log from gonotego.common import status from gonotego.settings import settings from gonotego.uploader.email import email_uploader @@ -68,8 +69,17 @@ def main(): status.set(Status.UPLOADER_READY, True) last_upload = None + last_cleanup = None while True: + # Once a day, clean up old local note copies -- and even then, only files + # older than ~6 months, and only if the disk is under real pressure. + if last_cleanup is None or time.time() - last_cleanup > 24 * 60 * 60: + last_cleanup = time.time() + deleted = note_log.cleanup() + if deleted: + print(f'Disk pressure: cleaned up {len(deleted)} old local note files.') + # Don't even try uploading notes if we don't have a connection. internet.wait_for_internet(on_disconnect=uploader.handle_disconnect) @@ -105,6 +115,9 @@ def main(): print('Upload unsuccessful.') status.set(Status.UPLOADER_ACTIVE, False) + # Only remove note events from the queue when the upload succeeded. + # On failure the events stay queued for the next attempt, and either + # way every note also remains in the durable local note log. if upload_successful: for note_event_bytes in note_event_bytes_list: note_events_queue.commit(note_event_bytes) From 293b7f2a5b42a187fcf7e0e7457bfc3f0c5f96b5 Mon Sep 17 00:00:00 2001 From: David Bieber Date: Thu, 16 Jul 2026 21:46:24 +0000 Subject: [PATCH 2/2] Add :xtime/:rxtime alleged-time offsets for offline note timestamps Go Note Go has no realtime clock, so after a flight or a stretch offline the system clock (and therefore note timestamps) can be wrong. - ':xtime 3:30pm' records an alleged-time offset: the difference between what time it really is and the system clock. Bare clock times snap to the nearest occurrence (+/-12h), so ':xtime 1am' at 8pm means tonight. ':xtime' says the current alleged time; ':xtime clear' resets. - Note events now carry BOTH the raw clock timestamp and the offset (NoteEvent.offset, effective_timestamp), in the queues and in the durable note log, so either time can be recovered later. - ':rxtime 4' retroactively applies the current offset to the last 4 hours of pending (not-yet-uploaded) notes; bare ':rxtime' reaches back to the last boot (/proc/uptime). - The Roam uploader titles new sessions with the first note's effective time (clock + offset) instead of the upload time. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_016ib1Yk692v5GKFz1bg5hTW --- gonotego/command_center/commands.py | 1 + gonotego/command_center/note_commands.py | 2 + gonotego/command_center/time_commands.py | 81 +++++++++++++++ gonotego/common/events.py | 12 +++ gonotego/common/interprocess.py | 15 +++ gonotego/common/note_log.py | 4 + gonotego/common/test_events.py | 19 ++++ gonotego/common/test_time_offsets.py | 120 +++++++++++++++++++++++ gonotego/common/time_offsets.py | 118 ++++++++++++++++++++++ gonotego/text/shell.py | 2 + gonotego/transcription/runner.py | 2 + gonotego/uploader/roam/roam_uploader.py | 10 +- 12 files changed, 383 insertions(+), 3 deletions(-) create mode 100644 gonotego/command_center/time_commands.py create mode 100644 gonotego/common/test_time_offsets.py create mode 100644 gonotego/common/time_offsets.py diff --git a/gonotego/command_center/commands.py b/gonotego/command_center/commands.py index 8f3ebeaa..33d426c2 100644 --- a/gonotego/command_center/commands.py +++ b/gonotego/command_center/commands.py @@ -10,6 +10,7 @@ from gonotego.command_center import note_commands # noqa: F401 from gonotego.command_center import settings_commands # noqa: F401 from gonotego.command_center import system_commands # noqa: F401 +from gonotego.command_center import time_commands # noqa: F401 from gonotego.command_center import twitter_commands # noqa: F401 from gonotego.command_center import wifi_commands # noqa: F401 from gonotego.command_center import registry diff --git a/gonotego/command_center/note_commands.py b/gonotego/command_center/note_commands.py index abc3b22a..1168c750 100644 --- a/gonotego/command_center/note_commands.py +++ b/gonotego/command_center/note_commands.py @@ -5,6 +5,7 @@ from gonotego.common import events from gonotego.common import interprocess from gonotego.common import note_log +from gonotego.common import time_offsets register_command = registry.register_command @@ -16,6 +17,7 @@ def get_timestamp(): def put_note_event(note_event): """Logs a note event locally and enqueues it for the uploader.""" + note_event.offset = time_offsets.get_offset() note_log.log(note_event) interprocess.get_note_events_queue().put(bytes(note_event)) interprocess.get_note_events_session_queue().put(bytes(note_event)) diff --git a/gonotego/command_center/time_commands.py b/gonotego/command_center/time_commands.py new file mode 100644 index 00000000..a4b47a42 --- /dev/null +++ b/gonotego/command_center/time_commands.py @@ -0,0 +1,81 @@ +"""Commands for correcting note timestamps when the system clock is wrong. + +Go Note Go has no realtime clock, so after a flight (or a long stretch +offline) the system clock can be off. These commands manage an +'alleged time' offset: + + :xtime 3:30pm -- declare what time it really is. Future notes carry both + the raw clock time and the offset to the alleged time. + :xtime -- say the current alleged time and offset. + :xtime clear -- clear the offset. + :rxtime 4 -- retroactively apply the current offset to the last + 4 hours of not-yet-uploaded notes. + :rxtime -- same, reaching back to the last boot. +""" + +from datetime import datetime +import time + +from gonotego.command_center import registry +from gonotego.command_center import system_commands +from gonotego.common import time_offsets + +register_command = registry.register_command + +CLEAR_WORDS = ('clear', 'off', 'reset', 'none') + + +@register_command('xtime {}') +def set_alleged_time(time_str): + """Sets the alleged time, recording its offset from the system clock.""" + time_str = time_str.strip() + if time_str.lower() in CLEAR_WORDS: + time_offsets.clear_offset() + system_commands.say('time offset cleared') + return + alleged_dt = time_offsets.parse_alleged_time(time_str) + if alleged_dt is None: + system_commands.say(f'could not parse time {time_str}') + return + offset = time_offsets.compute_offset(alleged_dt) + time_offsets.set_offset(offset) + say_alleged_time(prefix=f'offset {time_offsets.describe_offset(offset)}. ') + + +@register_command('xtime') +def say_alleged_time(prefix=''): + """Says the current alleged time and offset.""" + offset = time_offsets.get_offset() + alleged = datetime.fromtimestamp(time.time() + offset) + time_str = alleged.strftime('%A %l:%M %p').strip() + system_commands.say(f'{prefix}alleged time {time_str}') + + +@register_command('rxtime {}') +def apply_offset_to_recent_notes(hours_str): + """Retroactively applies the offset to the last N hours of pending notes.""" + hours_str = hours_str.strip() + try: + hours = float(hours_str) + except ValueError: + system_commands.say(f'could not parse hours {hours_str}') + return + cutoff = time.time() - hours * 60 * 60 + apply_offset_since(cutoff) + + +@register_command('rxtime') +def apply_offset_to_notes_since_boot(): + """Retroactively applies the offset to pending notes since the last boot.""" + cutoff = time_offsets.boot_timestamp() + if cutoff is None: + system_commands.say('could not determine boot time') + return + apply_offset_since(cutoff) + + +def apply_offset_since(cutoff): + offset = time_offsets.get_offset() + updated = time_offsets.apply_offset_retroactively(offset, cutoff) + system_commands.say( + f'applied offset {time_offsets.describe_offset(offset)} to {updated} pending notes') diff --git a/gonotego/common/events.py b/gonotego/common/events.py index 2c4c0d7d..1be4887d 100644 --- a/gonotego/common/events.py +++ b/gonotego/common/events.py @@ -46,6 +46,18 @@ class NoteEvent: action: Text audio_filepath: Text timestamp: datetime + # Seconds to add to the raw system-clock timestamp to get the alleged time. + # Set via ':xtime' when the system clock is known to be wrong (e.g. no + # internet while traveling). Notes carry both the raw timestamp and the + # offset, so either time can be recovered later. + offset: float = 0.0 + + @property + def effective_timestamp(self): + """The timestamp with the alleged-time offset applied.""" + if self.timestamp is None: + return None + return self.timestamp + (self.offset or 0.0) def __bytes__(self): return json.dumps(dataclasses.asdict(self)).encode('utf-8') diff --git a/gonotego/common/interprocess.py b/gonotego/common/interprocess.py index ea3710c4..27ef0b08 100644 --- a/gonotego/common/interprocess.py +++ b/gonotego/common/interprocess.py @@ -48,6 +48,21 @@ def commit(self, value): assert self.index >= 0 assert value == pop_value + def update_items(self, update_fn): + """Applies update_fn to every queued item, in place. + + update_fn receives the item's bytes and returns replacement bytes, or + None to leave the item unchanged. Returns the number of items updated. + """ + updated = 0 + values = self.r.lrange(self.key, 0, -1) + for i, value in enumerate(values): + new_value = update_fn(value) + if new_value is not None and new_value != value: + self.r.lset(self.key, i, new_value) + updated += 1 + return updated + def size(self): return self.r.llen(self.key) - self.index diff --git a/gonotego/common/note_log.py b/gonotego/common/note_log.py index 08e51867..759fe727 100644 --- a/gonotego/common/note_log.py +++ b/gonotego/common/note_log.py @@ -35,12 +35,16 @@ def log(note_event): timestamp = note_event.timestamp dt = datetime.fromtimestamp(timestamp) if timestamp else datetime.now() filepath = os.path.join(NOTE_LOG_DIR, dt.strftime('%Y-%m-%d') + '.jsonl') + offset = getattr(note_event, 'offset', 0.0) or 0.0 + effective_dt = datetime.fromtimestamp(timestamp + offset) if timestamp else dt entry = { 'text': note_event.text, 'action': note_event.action, 'audio_filepath': note_event.audio_filepath, 'timestamp': timestamp, 'time': dt.isoformat(), + 'offset': offset, + 'effective_time': effective_dt.isoformat(), } with open(filepath, 'a') as f: f.write(json.dumps(entry) + '\n') diff --git a/gonotego/common/test_events.py b/gonotego/common/test_events.py index 4823cf2d..de4aed19 100644 --- a/gonotego/common/test_events.py +++ b/gonotego/common/test_events.py @@ -29,6 +29,25 @@ def test_note_event(self): event2 = events.NoteEvent.from_bytes(event_bytes) self.assertEqual(event, event2) + def test_note_event_with_offset(self): + event = events.NoteEvent( + text='Example note.', + action=events.SUBMIT, + audio_filepath='/tmp/audio', + timestamp=1000.0, + offset=3600.0) + event_bytes = bytes(event) + event2 = events.NoteEvent.from_bytes(event_bytes) + self.assertEqual(event, event2) + self.assertEqual(event2.effective_timestamp, 4600.0) + + def test_note_event_from_bytes_without_offset(self): + # Events serialized before the offset field existed still deserialize. + event_bytes = b'{"text": "Old note.", "action": "submit", "audio_filepath": null, "timestamp": 1000.0}' + event = events.NoteEvent.from_bytes(event_bytes) + self.assertEqual(event.offset, 0.0) + self.assertEqual(event.effective_timestamp, 1000.0) + def test_led_event(self): event = events.LEDEvent( color=[0, 0, 0], diff --git a/gonotego/common/test_time_offsets.py b/gonotego/common/test_time_offsets.py new file mode 100644 index 00000000..a2ec975c --- /dev/null +++ b/gonotego/common/test_time_offsets.py @@ -0,0 +1,120 @@ +from datetime import datetime +import unittest +from unittest import mock + +from gonotego.common import events +from gonotego.common import interprocess +from gonotego.common import time_offsets + + +def make_note_event_bytes(text, timestamp, offset=0.0): + return bytes(events.NoteEvent( + text=text, + action=events.SUBMIT, + audio_filepath=None, + timestamp=timestamp, + offset=offset)) + + +class TimeOffsetsTest(unittest.TestCase): + + def test_compute_offset(self): + now = datetime(2026, 7, 16, 12, 0, 0).timestamp() + alleged = datetime(2026, 7, 16, 15, 30, 0) + self.assertEqual(time_offsets.compute_offset(alleged, now=now), 3.5 * 60 * 60) + + def test_compute_offset_negative(self): + now = datetime(2026, 7, 16, 12, 0, 0).timestamp() + alleged = datetime(2026, 7, 16, 11, 0, 0) + self.assertEqual(time_offsets.compute_offset(alleged, now=now), -60 * 60) + + def test_parse_alleged_time(self): + now = datetime(2026, 7, 16, 8, 0, 0).timestamp() + dt = time_offsets.parse_alleged_time('3:30pm', now=now) + self.assertIsNotNone(dt) + self.assertEqual((dt.hour, dt.minute), (15, 30)) + + def test_parse_alleged_time_snaps_to_nearest_occurrence(self): + # At 8pm, ':xtime 1:00am' means 1am tonight (5h ahead), not 1am this + # morning (19h ago). + now = datetime(2026, 7, 16, 20, 0, 0).timestamp() + dt = time_offsets.parse_alleged_time('1:00am', now=now) + self.assertEqual(dt, datetime(2026, 7, 17, 1, 0, 0)) + + # At 3am, ':xtime 11:00pm' means 11pm yesterday (4h ago). + now = datetime(2026, 7, 16, 3, 0, 0).timestamp() + dt = time_offsets.parse_alleged_time('11:00pm', now=now) + self.assertEqual(dt, datetime(2026, 7, 15, 23, 0, 0)) + + def test_parse_alleged_time_with_explicit_date_is_not_snapped(self): + now = datetime(2026, 7, 16, 20, 0, 0).timestamp() + dt = time_offsets.parse_alleged_time('july 20 3pm', now=now) + self.assertEqual(dt, datetime(2026, 7, 20, 15, 0, 0)) + + def test_parse_alleged_time_unparseable(self): + now = datetime(2026, 7, 16, 8, 0, 0).timestamp() + self.assertIsNone(time_offsets.parse_alleged_time('gobbledygook@#$', now=now)) + + def test_describe_offset(self): + self.assertEqual(time_offsets.describe_offset(3.5 * 60 * 60), '+3h30m00s') + self.assertEqual(time_offsets.describe_offset(-90), '-0h01m30s') + self.assertEqual(time_offsets.describe_offset(0), '+0h00m00s') + + def test_update_note_event_bytes_within_window(self): + note_event_bytes = make_note_event_bytes('recent', timestamp=1000.0) + updated = time_offsets.update_note_event_bytes(note_event_bytes, offset=120.0, cutoff=500.0) + self.assertIsNotNone(updated) + event = events.NoteEvent.from_bytes(updated) + self.assertEqual(event.offset, 120.0) + self.assertEqual(event.effective_timestamp, 1120.0) + self.assertEqual(event.text, 'recent') + + def test_update_note_event_bytes_before_cutoff(self): + note_event_bytes = make_note_event_bytes('old', timestamp=100.0) + self.assertIsNone( + time_offsets.update_note_event_bytes(note_event_bytes, offset=120.0, cutoff=500.0)) + + def test_apply_offset_retroactively_rewrites_pending_events(self): + values = [ + make_note_event_bytes('before cutoff', timestamp=100.0), + make_note_event_bytes('after cutoff', timestamp=1000.0), + make_note_event_bytes('also after', timestamp=2000.0), + ] + r = mock.MagicMock() + r.lrange.return_value = list(values) + with mock.patch.object(interprocess, 'get_redis_client', return_value=r): + queue = interprocess.InterprocessQueue('test_queue') + updated = time_offsets.apply_offset_retroactively(60.0, cutoff=500.0, queues=[queue]) + + self.assertEqual(updated, 2) + lset_calls = r.lset.call_args_list + self.assertEqual(len(lset_calls), 2) + # The event before the cutoff (index 0) is untouched. + self.assertEqual([call.args[1] for call in lset_calls], [1, 2]) + for call in lset_calls: + event = events.NoteEvent.from_bytes(call.args[2]) + self.assertEqual(event.offset, 60.0) + + def test_apply_offset_retroactively_already_offset_events_are_updated(self): + values = [make_note_event_bytes('note', timestamp=1000.0, offset=30.0)] + r = mock.MagicMock() + r.lrange.return_value = list(values) + with mock.patch.object(interprocess, 'get_redis_client', return_value=r): + queue = interprocess.InterprocessQueue('test_queue') + updated = time_offsets.apply_offset_retroactively(60.0, cutoff=500.0, queues=[queue]) + self.assertEqual(updated, 1) + event = events.NoteEvent.from_bytes(r.lset.call_args.args[2]) + self.assertEqual(event.offset, 60.0) + + def test_update_items_no_changes(self): + r = mock.MagicMock() + r.lrange.return_value = [b'a', b'b'] + with mock.patch.object(interprocess, 'get_redis_client', return_value=r): + queue = interprocess.InterprocessQueue('test_queue') + updated = queue.update_items(lambda value: None) + self.assertEqual(updated, 0) + r.lset.assert_not_called() + + +if __name__ == '__main__': + unittest.main() diff --git a/gonotego/common/time_offsets.py b/gonotego/common/time_offsets.py new file mode 100644 index 00000000..4d7c00b0 --- /dev/null +++ b/gonotego/common/time_offsets.py @@ -0,0 +1,118 @@ +"""Support for an 'alleged time' offset while offline. + +Go Note Go devices have no realtime clock, so after a flight or a stretch +offline the system clock can be wrong. ':xtime