feat(DM01-6260): optimize console log loading with gzip compression and progressive rendering

infrabox/generator/deployments.json

@@ -502,53 +502,6 @@
                 }
             ]
         },
-        {
-            "type": "docker",
-            "build_context": "../..",
-            "name": "service-aks",
-            "docker_file": "src/services/aks/Dockerfile",
-            "resources": {
-                "limits": {
-                    "cpu": 1,
-                    "memory": 2048
-                }
-            },
-            "deployments": [
-                {
-                    "type": "docker-registry",
-                    "host": "quay.io/infrabox",
-                    "repository": "service-aks",
-                    "username": "infrabox+infrabox_ci",
-                    "password": {
-                        "$secret": "QUAY_PASSWORD"
-                    }
-                }
-            ]
-        },
-        {
-            "type": "docker",
-            "build_context": "../..",
-            "name": "service-gardener",
-            "docker_file": "src/services/gardener/Dockerfile",
-            "resources": {
-                "limits": {
-                    "cpu": 1,
-                    "memory": 2048
-                }
-            },
-            "deployments": [
-                {
-                    "type": "docker-registry",
-                    "host": "quay.io/infrabox",
-                    "repository": "service-gardener",
-                    "username": "infrabox+infrabox_ci",
-                    "password": {
-                        "$secret": "QUAY_PASSWORD"
-                    }
-                }
-            ]
-        },
-
         {
             "type": "docker",
             "build_context": "../..",

src/api/handlers/projects/jobs.py

@@ -3,6 +3,7 @@
 import uuid
 import re
 import mimetypes
+import gzip as gzip_module
 
 from io import BytesIO
 
@@ -776,39 +777,88 @@ def get(self, project_id, job_id):
         '''
         return redirect("/api/v1/projects/%s/jobs/%s/archive/download?filename=all_archives.tar.gz" %(project_id, job_id))
 
+def _console_response(output):
+    accepts_gzip = 'gzip' in request.headers.get('Accept-Encoding', '')
+    if accepts_gzip and len(output.encode('utf-8')) > 102400:
+        compressed = gzip_module.compress(output.encode('utf-8'), compresslevel=6)
+        resp = Response(compressed, mimetype='text/plain')
+        resp.headers['Content-Encoding'] = 'gzip'
+        resp.headers['Content-Length'] = len(compressed)
+        return resp
+    return Response(output, mimetype='text/plain')
+
+
 @ns.route('/<job_id>/console')
 @api.response(403, 'Not Authorized')
 class Console(Resource):
-
     def get(self, project_id, job_id):
         '''
-        Returns job's console output
+        Returns job's console output. Pass ?tail=N to return only the last N lines.
         '''
-        result = g.db.execute_one_dict('''
-            SELECT console
-            FROM job
-            WHERE   id = %s
-                AND project_id = %s
-        ''', [job_id, project_id])
-
-        if result and result['console']:
-            return Response(result['console'], mimetype='text/plain')
+        tail = request.args.get('tail', None)
+        if tail is not None:
+            try:
+                tail = max(1, int(tail))
+            except (ValueError, TypeError):
+                tail = None
 
-        result = g.db.execute_many_dict('''
-            SELECT output
-            FROM console
-            WHERE job_id = %s
-            ORDER BY date
-        ''', [job_id])
+        if tail:
+            result = g.db.execute_one_dict('''
+                SELECT array_to_string(
+                    (string_to_array(console, E'\\n'))
+                    [greatest(array_length(string_to_array(console, E'\\n'), 1) - %s + 1, 1):],
+                    E'\\n'
+                ) AS console
+                FROM job
+                WHERE id = %s AND project_id = %s
+                  AND console IS NOT NULL
+                  AND console != ''
+                  AND console != 'deleted'
+            ''', [tail, job_id, project_id])
+        else:
+            result = g.db.execute_one_dict('''
+                SELECT console
+                FROM job
+                WHERE id = %s AND project_id = %s
+                  AND console != 'deleted'
+            ''', [job_id, project_id])
 
-        if not result:
+        if result and result['console']:
+            return _console_response(result['console'])
+
+        # console table stores chunks (one row per flush), not individual lines.
+        # Fetch enough chunks to cover tail lines, then trim to exact line count.
+        if tail:
+            chunk_limit = max(tail, 200)
+            rows = g.db.execute_many_dict('''
+                SELECT output FROM (
+                    SELECT output, date
+                    FROM console
+                    WHERE job_id = %s
+                      AND job_id IN (SELECT id FROM job WHERE project_id = %s)
+                    ORDER BY date DESC
+                    LIMIT %s
+                ) sub
+                ORDER BY date
+            ''', [job_id, project_id, chunk_limit])
+        else:
+            rows = g.db.execute_many_dict('''
+                SELECT output
+                FROM console
+                WHERE job_id = %s
+                  AND job_id IN (SELECT id FROM job WHERE project_id = %s)
+                ORDER BY date
+            ''', [job_id, project_id])
+
+        if not rows:
             return ''
 
-        output = ''
-        for r in result:
-            output += r['output']
-
-        return Response(output, mimetype='text/plain')
+        output = ''.join(r['output'] for r in rows)
+        if tail:
+            lines = output.split('\n')
+            if len(lines) > tail:
+                output = '\n'.join(lines[-tail:])
+        return _console_response(output)
 
 
 @ns.route('/<job_id>/output', doc=False)

src/dashboard-client/src/models/Job.js

@@ -105,6 +105,7 @@ export default class Job {
         this.currentSection = null
         this.linesProcessed = 0
         this.hasLogsAvailable = false
+        this._consoleFetched = false
         this.message = message
         this.definition = definition
         this.nodeName = nodeName
@@ -239,20 +240,39 @@ export default class Job {
     }
 
     loadConsole () {
-        if (this.sections.length) {
+        if (this._consoleFetched) {
             return
         }
+        this._consoleFetched = true
 
-        return NewAPIService.get(`projects/${this.project.id}/jobs/${this.id}/console`)
+        return NewAPIService.get(`projects/${this.project.id}/jobs/${this.id}/console?tail=500`)
             .then((console) => {
                 store.commit('setConsole', { job: this, console: console })
                 events.listenConsole(this.id)
+                this._prefetchFullConsole()
             })
             .catch((err) => {
+                this._consoleFetched = false
                 NotificationService.$emit('NOTIFICATION', new Notification(err))
             })
     }
 
+    _prefetchFullConsole () {
+        if (this.state !== 'finished') {
+            return
+        }
+        NewAPIService.get(`projects/${this.project.id}/jobs/${this.id}/console`)
+            .then((console) => {
+                if (console) {
+                    this.sections = []
+                    this.currentSection = null
+                    this.linesProcessed = 0
+                    store.commit('setConsole', { job: this, console: console })
+                }
+            })
+            .catch(() => {})
+    }
+
     loadTabs () {
         return NewAPIService.get(`projects/${this.project.id}/jobs/${this.id}/tabs`)
             .then((tabs) => {