diff --git a/vulnerabilities/api_v2.py b/vulnerabilities/api_v2.py index 6e0ab9213..d16fbf61c 100644 --- a/vulnerabilities/api_v2.py +++ b/vulnerabilities/api_v2.py @@ -788,7 +788,7 @@ def get_latest_run(self, schedule): def to_representation(self, schedule): representation = super().to_representation(schedule) - representation["run_interval"] = f"{schedule.run_interval}hr" + representation["run_interval"] = f"{schedule.run_interval}min" representation["execution_timeout"] = f"{schedule.execution_timeout}hr" return representation diff --git a/vulnerabilities/importers/__init__.py b/vulnerabilities/importers/__init__.py index 5ff64a1ff..e5d28f6ff 100644 --- a/vulnerabilities/importers/__init__.py +++ b/vulnerabilities/importers/__init__.py @@ -159,37 +159,37 @@ collect_fix_commits_v2.CollectGitFixCommitsPipeline, collect_fix_commits_v2.CollectJenkinsFixCommitsPipeline, collect_fix_commits_v2.CollectGitlabFixCommitsPipeline, - github_importer.GitHubAPIImporterPipeline, - gitlab_importer.GitLabImporterPipeline, - github_osv.GithubOSVImporter, - pypa_importer.PyPaImporterPipeline, - npm_importer.NpmImporterPipeline, - nginx_importer.NginxImporterPipeline, - pysec_importer.PyPIImporterPipeline, - apache_tomcat.ApacheTomcatImporter, - postgresql.PostgreSQLImporter, - debian.DebianImporter, - curl.CurlImporter, - epss.EPSSImporter, - vulnrichment.VulnrichImporter, - alpine_linux_importer.AlpineLinuxImporterPipeline, - apache_kafka.ApacheKafkaImporter, - ruby.RubyImporter, - redhat.RedhatImporter, - archlinux.ArchlinuxImporter, - debian_oval.DebianOvalImporter, - retiredotnet.RetireDotnetImporter, - apache_httpd.ApacheHTTPDImporter, - mozilla.MozillaImporter, - gentoo.GentooImporter, - istio.IstioImporter, - project_kb_msr2019.ProjectKBMSRImporter, - suse_scores.SUSESeverityScoreImporter, - elixir_security.ElixirSecurityImporter, - xen.XenImporter, - ubuntu_usn.UbuntuUSNImporter, - fireeye.FireyeImporter, - oss_fuzz.OSSFuzzImporter, + # github_importer.GitHubAPIImporterPipeline, + # gitlab_importer.GitLabImporterPipeline, + # github_osv.GithubOSVImporter, + # pypa_importer.PyPaImporterPipeline, + # npm_importer.NpmImporterPipeline, + # nginx_importer.NginxImporterPipeline, + # pysec_importer.PyPIImporterPipeline, + # apache_tomcat.ApacheTomcatImporter, + # postgresql.PostgreSQLImporter, + # debian.DebianImporter, + # curl.CurlImporter, + # epss.EPSSImporter, + # vulnrichment.VulnrichImporter, + # alpine_linux_importer.AlpineLinuxImporterPipeline, + # apache_kafka.ApacheKafkaImporter, + # ruby.RubyImporter, + # redhat.RedhatImporter, + # archlinux.ArchlinuxImporter, + # debian_oval.DebianOvalImporter, + # retiredotnet.RetireDotnetImporter, + # apache_httpd.ApacheHTTPDImporter, + # mozilla.MozillaImporter, + # gentoo.GentooImporter, + # istio.IstioImporter, + # project_kb_msr2019.ProjectKBMSRImporter, + # suse_scores.SUSESeverityScoreImporter, + # elixir_security.ElixirSecurityImporter, + # xen.XenImporter, + # ubuntu_usn.UbuntuUSNImporter, + # fireeye.FireyeImporter, + # oss_fuzz.OSSFuzzImporter, ] ) diff --git a/vulnerabilities/improvers/__init__.py b/vulnerabilities/improvers/__init__.py index 05d08cbbe..138ca59c7 100644 --- a/vulnerabilities/improvers/__init__.py +++ b/vulnerabilities/improvers/__init__.py @@ -16,12 +16,14 @@ from vulnerabilities.pipelines import enhance_with_metasploit from vulnerabilities.pipelines import flag_ghost_packages from vulnerabilities.pipelines import populate_vulnerability_summary_pipeline -from vulnerabilities.pipelines.v2_improvers import archive_urls -from vulnerabilities.pipelines.v2_improvers import collect_ssvc_trees +from vulnerabilities.pipelines.v2_improvers import archive_urls as archive_urls_v2 +from vulnerabilities.pipelines.v2_improvers import collect_ssvc_trees as collect_ssvc_trees_v2 from vulnerabilities.pipelines.v2_improvers import compute_advisory_todo as compute_advisory_todo_v2 from vulnerabilities.pipelines.v2_improvers import compute_package_risk as compute_package_risk_v2 from vulnerabilities.pipelines.v2_improvers import enhance_with_exploitdb as exploitdb_v2 -from vulnerabilities.pipelines.v2_improvers import enhance_with_github_poc +from vulnerabilities.pipelines.v2_improvers import ( + enhance_with_github_poc as enhance_with_github_poc_v2, +) from vulnerabilities.pipelines.v2_improvers import enhance_with_kev as enhance_with_kev_v2 from vulnerabilities.pipelines.v2_improvers import ( enhance_with_metasploit as enhance_with_metasploit_v2, @@ -30,9 +32,13 @@ from vulnerabilities.pipelines.v2_improvers import ( group_advisories_for_packages as group_advisories_for_packages_v2, ) -from vulnerabilities.pipelines.v2_improvers import mark_unfurl_version_range -from vulnerabilities.pipelines.v2_improvers import reference_collect_commits -from vulnerabilities.pipelines.v2_improvers import relate_severities +from vulnerabilities.pipelines.v2_improvers import ( + mark_unfurl_version_range as mark_unfurl_version_range_v2, +) +from vulnerabilities.pipelines.v2_improvers import ( + reference_collect_commits as reference_collect_commits_v2, +) +from vulnerabilities.pipelines.v2_improvers import relate_severities as relate_severities_v2 from vulnerabilities.pipelines.v2_improvers import unfurl_version_range as unfurl_version_range_v2 from vulnerabilities.utils import create_registry @@ -44,36 +50,36 @@ enhance_with_metasploit_v2.MetasploitImproverPipeline, compute_package_risk_v2.ComputePackageRiskPipeline, unfurl_version_range_v2.UnfurlVersionRangePipeline, - collect_ssvc_trees.CollectSSVCPipeline, - relate_severities.RelateSeveritiesPipeline, - archive_urls.ArchiveImproverPipeline, + collect_ssvc_trees_v2.CollectSSVCPipeline, + relate_severities_v2.RelateSeveritiesPipeline, + archive_urls_v2.ArchiveImproverPipeline, compute_advisory_todo_v2.ComputeToDo, - reference_collect_commits.CollectReferencesFixCommitsPipeline, - enhance_with_github_poc.GithubPocsImproverPipeline, - mark_unfurl_version_range.MarkUnfurlVersionRangePipeline, - valid_versions.GitHubBasicImprover, - valid_versions.GitLabBasicImprover, - valid_versions.NginxBasicImprover, - valid_versions.ApacheHTTPDImprover, - valid_versions.DebianBasicImprover, - valid_versions.NpmImprover, - valid_versions.ElixirImprover, - valid_versions.ApacheTomcatImprover, - valid_versions.ApacheKafkaImprover, - valid_versions.IstioImprover, - valid_versions.DebianOvalImprover, - valid_versions.OSSFuzzImprover, - valid_versions.RubyImprover, - valid_versions.GithubOSVImprover, - vulnerability_status.VulnerabilityStatusImprover, - valid_versions.CurlImprover, - flag_ghost_packages.FlagGhostPackagePipeline, - enhance_with_kev.VulnerabilityKevPipeline, - enhance_with_metasploit.MetasploitImproverPipeline, - enhance_with_exploitdb.ExploitDBImproverPipeline, - compute_package_risk.ComputePackageRiskPipeline, - compute_package_version_rank.ComputeVersionRankPipeline, - populate_vulnerability_summary_pipeline.PopulateVulnerabilitySummariesPipeline, + reference_collect_commits_v2.CollectReferencesFixCommitsPipeline, + enhance_with_github_poc_v2.GithubPocsImproverPipeline, + mark_unfurl_version_range_v2.MarkUnfurlVersionRangePipeline, group_advisories_for_packages_v2.GroupAdvisoriesForPackages, + # valid_versions.GitHubBasicImprover, + # valid_versions.GitLabBasicImprover, + # valid_versions.NginxBasicImprover, + # valid_versions.ApacheHTTPDImprover, + # valid_versions.DebianBasicImprover, + # valid_versions.NpmImprover, + # valid_versions.ElixirImprover, + # valid_versions.ApacheTomcatImprover, + # valid_versions.ApacheKafkaImprover, + # valid_versions.IstioImprover, + # valid_versions.DebianOvalImprover, + # valid_versions.OSSFuzzImprover, + # valid_versions.RubyImprover, + # valid_versions.GithubOSVImprover, + # vulnerability_status.VulnerabilityStatusImprover, + # valid_versions.CurlImprover, + # flag_ghost_packages.FlagGhostPackagePipeline, + # enhance_with_kev.VulnerabilityKevPipeline, + # enhance_with_metasploit.MetasploitImproverPipeline, + # enhance_with_exploitdb.ExploitDBImproverPipeline, + # compute_package_risk.ComputePackageRiskPipeline, + # compute_package_version_rank.ComputeVersionRankPipeline, + # populate_vulnerability_summary_pipeline.PopulateVulnerabilitySummariesPipeline, ] ) diff --git a/vulnerabilities/migrations/0137_alter_pipelineschedule_run_interval.py b/vulnerabilities/migrations/0137_alter_pipelineschedule_run_interval.py new file mode 100644 index 000000000..a861b866b --- /dev/null +++ b/vulnerabilities/migrations/0137_alter_pipelineschedule_run_interval.py @@ -0,0 +1,46 @@ +# Generated by Django 5.2.11 on 2026-06-16 12:29 + +import django.core.validators +from django.db import migrations +from django.db import models +from django.db.models import F + + +class Migration(migrations.Migration): + + dependencies = [ + ("vulnerabilities", "0136_advisorysetmember_unique_advisory_per_set"), + ] + + def convert_hours_to_minutes(apps, schema_editor): + PipelineSchedule = apps.get_model("vulnerabilities", "PipelineSchedule") + PipelineSchedule.objects.update(run_interval=F("run_interval") * 60) + + def revert_convert_hours_to_minutes(apps, schema_editor): + PipelineSchedule = apps.get_model("vulnerabilities", "PipelineSchedule") + for schedule in PipelineSchedule.objects.all(): + schedule.run_interval = min(8760, max(1, schedule.run_interval // 60)) + schedule.save(update_fields=["run_interval"]) + + operations = [ + migrations.AlterField( + model_name="pipelineschedule", + name="run_interval", + field=models.IntegerField( + default=720, + help_text="Number of minutes to wait between run of this pipeline.", + validators=[ + django.core.validators.MinValueValidator( + 5, message="Interval must be at least 5 minutes." + ), + django.core.validators.MaxValueValidator( + 43200, message="Interval must be at most 43200 minutes (i.e 30 days)." + ), + ], + ), + ), + migrations.RunPython( + convert_hours_to_minutes, + revert_convert_hours_to_minutes, + ), + ] diff --git a/vulnerabilities/models.py b/vulnerabilities/models.py index af0e0df78..8c7a114d9 100644 --- a/vulnerabilities/models.py +++ b/vulnerabilities/models.py @@ -2302,13 +2302,15 @@ class ExecutionPriority(models.IntegerChoices): ), ) - run_interval = models.PositiveSmallIntegerField( + run_interval = models.IntegerField( validators=[ - MinValueValidator(1, message="Interval must be at least 1 hour."), - MaxValueValidator(8760, message="Interval must be at most 8760 hours."), + MinValueValidator(5, message="Interval must be at least 5 minutes."), + MaxValueValidator( + 43200, message="Interval must be at most 43200 minutes (i.e 30 days)." + ), ], - default=24, - help_text=("Number of hours to wait between run of this pipeline."), + default=720, + help_text=("Number of minutes to wait between run of this pipeline."), ) run_priority = models.IntegerField( diff --git a/vulnerabilities/pipelines/__init__.py b/vulnerabilities/pipelines/__init__.py index 873aede8b..b1ff9d1fe 100644 --- a/vulnerabilities/pipelines/__init__.py +++ b/vulnerabilities/pipelines/__init__.py @@ -145,8 +145,8 @@ class VulnerableCodePipeline(PipelineDefinition, BasePipelineRun): # When set to true pipeline is run only once. # To rerun onetime pipeline reset is_active field to True via migration. run_once = False - # Interval between runs in hour. - run_interval = 24 + # Interval between runs in minutes. + run_interval = 720 run_priority = PipelineSchedule.ExecutionPriority.DEFAULT def on_failure(self): @@ -180,8 +180,8 @@ class VulnerableCodeBaseImporterPipeline(VulnerableCodePipeline): # When set to true pipeline is run only once. # To rerun onetime pipeline reset is_active field to True via migration. run_once = False - # Interval between runs in hour. - run_interval = 24 + # Interval between runs in minutes. + run_interval = 720 run_priority = PipelineSchedule.ExecutionPriority.DEFAULT @classmethod @@ -290,8 +290,8 @@ class VulnerableCodeBaseImporterPipelineV2(VulnerableCodePipeline): # When set to true pipeline is run only once. # To rerun onetime pipeline reset is_active field to True via migration. run_once = False - # Interval between runs in hour. - run_interval = 24 + # Interval between runs in minutes. + run_interval = 720 run_priority = PipelineSchedule.ExecutionPriority.DEFAULT @classmethod diff --git a/vulnerabilities/pipelines/v2_improvers/collect_ssvc_trees.py b/vulnerabilities/pipelines/v2_improvers/collect_ssvc_trees.py index a0509c00b..02f7f4700 100644 --- a/vulnerabilities/pipelines/v2_improvers/collect_ssvc_trees.py +++ b/vulnerabilities/pipelines/v2_improvers/collect_ssvc_trees.py @@ -16,6 +16,7 @@ from vulnerabilities.models import SSVC from vulnerabilities.models import AdvisorySeverity from vulnerabilities.models import AdvisoryV2 +from vulnerabilities.models import PipelineSchedule from vulnerabilities.pipelines import VulnerableCodePipeline from vulnerabilities.severity_systems import SCORING_SYSTEMS @@ -31,6 +32,10 @@ class CollectSSVCPipeline(VulnerableCodePipeline): pipeline_id = "collect_ssvc_trees" + # Run pipeline every 30 minutes. + run_interval = 30 + run_priority = PipelineSchedule.ExecutionPriority.HIGH + @classmethod def steps(cls): return (cls.collect_ssvc_data,) diff --git a/vulnerabilities/pipelines/v2_improvers/compute_package_risk.py b/vulnerabilities/pipelines/v2_improvers/compute_package_risk.py index d97d637e1..b4b5cd235 100644 --- a/vulnerabilities/pipelines/v2_improvers/compute_package_risk.py +++ b/vulnerabilities/pipelines/v2_improvers/compute_package_risk.py @@ -15,6 +15,7 @@ from vulnerabilities.models import AdvisorySeverity from vulnerabilities.models import AdvisoryV2 from vulnerabilities.models import PackageV2 +from vulnerabilities.models import PipelineSchedule from vulnerabilities.pipelines import VulnerableCodePipeline from vulnerabilities.pipes.risk_score import bulk_update from vulnerabilities.risk import compute_vulnerability_risk_factors @@ -30,6 +31,10 @@ class ComputePackageRiskPipeline(VulnerableCodePipeline): pipeline_id = "compute_package_risk_v2" license_expression = None + # Run pipeline every 30 minutes. + run_interval = 30 + run_priority = PipelineSchedule.ExecutionPriority.HIGH + @classmethod def steps(cls): return ( diff --git a/vulnerabilities/pipelines/v2_improvers/enhance_with_exploitdb.py b/vulnerabilities/pipelines/v2_improvers/enhance_with_exploitdb.py index 561e1deff..79e0f9979 100644 --- a/vulnerabilities/pipelines/v2_improvers/enhance_with_exploitdb.py +++ b/vulnerabilities/pipelines/v2_improvers/enhance_with_exploitdb.py @@ -17,6 +17,7 @@ from dateutil import parser as dateparser from vulnerabilities.models import AdvisoryExploit +from vulnerabilities.models import PipelineSchedule from vulnerabilities.pipelines import VulnerableCodePipeline from vulnerabilities.utils import build_alias_to_advisory_map @@ -30,6 +31,10 @@ class ExploitDBImproverPipeline(VulnerableCodePipeline): pipeline_id = "enhance_with_exploitdb_v2" spdx_license_expression = "GPL-2.0" + # Run pipeline every 30 minutes. + run_interval = 30 + run_priority = PipelineSchedule.ExecutionPriority.HIGH + @classmethod def steps(cls): return ( diff --git a/vulnerabilities/pipelines/v2_improvers/enhance_with_github_poc.py b/vulnerabilities/pipelines/v2_improvers/enhance_with_github_poc.py index 28baa53ea..c155536f2 100644 --- a/vulnerabilities/pipelines/v2_improvers/enhance_with_github_poc.py +++ b/vulnerabilities/pipelines/v2_improvers/enhance_with_github_poc.py @@ -13,9 +13,7 @@ from aboutcode.pipeline import LoopProgress from fetchcode.vcs import fetch_via_vcs -from vulnerabilities.models import AdvisoryAlias from vulnerabilities.models import AdvisoryPOC -from vulnerabilities.models import AdvisoryV2 from vulnerabilities.pipelines import VulnerableCodePipeline from vulnerabilities.utils import relate_aliases_with_advisories diff --git a/vulnerabilities/pipelines/v2_improvers/enhance_with_kev.py b/vulnerabilities/pipelines/v2_improvers/enhance_with_kev.py index cc7721acd..2ea007d1f 100644 --- a/vulnerabilities/pipelines/v2_improvers/enhance_with_kev.py +++ b/vulnerabilities/pipelines/v2_improvers/enhance_with_kev.py @@ -14,6 +14,7 @@ from aboutcode.pipeline import LoopProgress from vulnerabilities.models import AdvisoryExploit +from vulnerabilities.models import PipelineSchedule from vulnerabilities.pipelines import VulnerableCodePipeline from vulnerabilities.utils import build_alias_to_advisory_map @@ -27,6 +28,10 @@ class VulnerabilityKevPipeline(VulnerableCodePipeline): pipeline_id = "enhance_with_kev_v2" license_expression = None + # Run pipeline every 30 minutes. + run_interval = 30 + run_priority = PipelineSchedule.ExecutionPriority.HIGH + @classmethod def steps(cls): return ( diff --git a/vulnerabilities/pipelines/v2_improvers/enhance_with_metasploit.py b/vulnerabilities/pipelines/v2_improvers/enhance_with_metasploit.py index bee3e47a7..b985afe21 100644 --- a/vulnerabilities/pipelines/v2_improvers/enhance_with_metasploit.py +++ b/vulnerabilities/pipelines/v2_improvers/enhance_with_metasploit.py @@ -16,6 +16,7 @@ from dateutil import parser as dateparser from vulnerabilities.models import AdvisoryExploit +from vulnerabilities.models import PipelineSchedule from vulnerabilities.pipelines import VulnerableCodePipeline from vulnerabilities.utils import build_alias_to_advisory_map @@ -29,6 +30,10 @@ class MetasploitImproverPipeline(VulnerableCodePipeline): pipeline_id = "enhance_with_metasploit_v2" spdx_license_expression = "BSD-3-clause" + # Run pipeline every 30 minutes. + run_interval = 30 + run_priority = PipelineSchedule.ExecutionPriority.HIGH + @classmethod def steps(cls): return ( diff --git a/vulnerabilities/pipelines/v2_improvers/flag_ghost_packages.py b/vulnerabilities/pipelines/v2_improvers/flag_ghost_packages.py index 8a4825df4..647f80871 100644 --- a/vulnerabilities/pipelines/v2_improvers/flag_ghost_packages.py +++ b/vulnerabilities/pipelines/v2_improvers/flag_ghost_packages.py @@ -25,6 +25,9 @@ class FlagGhostPackagePipeline(VulnerableCodePipeline): pipeline_id = "flag_ghost_packages_v2" + # Run pipeline every 6 hours. + run_interval = 360 + @classmethod def steps(cls): return (cls.flag_ghost_packages,) diff --git a/vulnerabilities/pipelines/v2_improvers/mark_unfurl_version_range.py b/vulnerabilities/pipelines/v2_improvers/mark_unfurl_version_range.py index 72d22a5f6..c2d6d55c2 100644 --- a/vulnerabilities/pipelines/v2_improvers/mark_unfurl_version_range.py +++ b/vulnerabilities/pipelines/v2_improvers/mark_unfurl_version_range.py @@ -24,7 +24,6 @@ from vulnerabilities.models import PackageV2 from vulnerabilities.models import PipelineSchedule from vulnerabilities.pipelines import VulnerableCodePipeline -from vulnerabilities.pipes.group_advisories import group_advisory_for_package from vulnerabilities.pipes.group_advisories import group_single_package_with_provided_advisories from vulnerabilities.pipes.risk_score import compute_package_risk_score_bulk from vulnerabilities.utils import TYPES_WITH_MULTIPLE_IMPORTERS @@ -37,7 +36,8 @@ class MarkUnfurlVersionRangePipeline(VulnerableCodePipeline): pipeline_id = "mark_unfurl_version_range_v2" - run_interval = 1 + # Run pipeline every 10 minutes. + run_interval = 10 run_priority = PipelineSchedule.ExecutionPriority.HIGH @classmethod diff --git a/vulnerabilities/pipelines/v2_improvers/relate_severities.py b/vulnerabilities/pipelines/v2_improvers/relate_severities.py index 8ef70445b..1d7083f98 100644 --- a/vulnerabilities/pipelines/v2_improvers/relate_severities.py +++ b/vulnerabilities/pipelines/v2_improvers/relate_severities.py @@ -13,6 +13,7 @@ from django.db import transaction from vulnerabilities.models import AdvisoryV2 +from vulnerabilities.models import PipelineSchedule from vulnerabilities.pipelines import VulnerableCodePipeline from vulnerabilities.pipelines.v2_importers.epss_importer_v2 import EPSSImporterPipeline from vulnerabilities.pipelines.v2_importers.suse_score_importer import ( @@ -35,6 +36,10 @@ class RelateSeveritiesPipeline(VulnerableCodePipeline): pipeline_id = "relate_severities_v2" + # Run pipeline every 30 minutes. + run_interval = 30 + run_priority = PipelineSchedule.ExecutionPriority.HIGH + # Severity systems to process SUPPORTED_SYSTEMS = { EPSS.identifier, diff --git a/vulnerabilities/pipelines/v2_improvers/unfurl_version_range.py b/vulnerabilities/pipelines/v2_improvers/unfurl_version_range.py index b099b26bf..ba0d0a1ce 100644 --- a/vulnerabilities/pipelines/v2_improvers/unfurl_version_range.py +++ b/vulnerabilities/pipelines/v2_improvers/unfurl_version_range.py @@ -21,17 +21,12 @@ from univers.version_range import RANGE_CLASS_BY_SCHEMES from univers.version_range import VersionRange -from vulnerabilities.models import AdvisoryV2 from vulnerabilities.models import ImpactedPackage from vulnerabilities.models import ImpactedPackageAffecting -from vulnerabilities.models import ImpactedPackageFixedBy from vulnerabilities.models import PackageV2 from vulnerabilities.models import PipelineSchedule from vulnerabilities.pipelines import VulnerableCodePipeline from vulnerabilities.pipes.fetchcode_utils import get_versions -from vulnerabilities.pipes.group_advisories import group_advisory_for_package -from vulnerabilities.pipes.risk_score import compute_package_risk_score_bulk -from vulnerabilities.utils import TYPES_WITH_MULTIPLE_IMPORTERS from vulnerabilities.utils import update_purl_version @@ -44,7 +39,8 @@ class UnfurlVersionRangePipeline(VulnerableCodePipeline): pipeline_id = "unfurl_version_range_v2" - run_interval = 1 + # Run pipeline every 10 minutes. + run_interval = 10 run_priority = PipelineSchedule.ExecutionPriority.HIGH # Days elapsed before version range is re-unfurled diff --git a/vulnerabilities/schedules.py b/vulnerabilities/schedules.py index 27fb7e09a..39836a3dd 100644 --- a/vulnerabilities/schedules.py +++ b/vulnerabilities/schedules.py @@ -29,7 +29,7 @@ def schedule_execution(pipeline_schedule, execute_now=False): if not execute_now: first_execution = pipeline_schedule.next_run_date - interval_in_seconds = pipeline_schedule.run_interval * 60 * 60 + interval_in_seconds = pipeline_schedule.run_interval * 60 job = scheduler.schedule( scheduled_time=first_execution, @@ -97,7 +97,7 @@ def update_pipeline_schedule(): PipelineSchedule.objects.exclude(pipeline_id__in=pipelines.keys()).delete() for id, pipeline_class in pipelines.items(): run_once = getattr(pipeline_class, "run_once", False) - run_interval = getattr(pipeline_class, "run_interval", 24) + run_interval = getattr(pipeline_class, "run_interval", 720) run_priority = getattr( pipeline_class, "run_priority", PipelineSchedule.ExecutionPriority.DEFAULT ) @@ -112,6 +112,7 @@ def update_pipeline_schedule(): ) if not created: - pipeline.run_priority = run_priority - pipeline.run_interval = run_interval - pipeline.save() + if pipeline.run_priority != run_priority or pipeline.run_interval != run_interval: + pipeline.run_priority = run_priority + pipeline.run_interval = run_interval + pipeline.save() diff --git a/vulnerabilities/tasks.py b/vulnerabilities/tasks.py index a78c13551..ee9ae00e2 100644 --- a/vulnerabilities/tasks.py +++ b/vulnerabilities/tasks.py @@ -179,7 +179,7 @@ def compute_queue_load_factor(): for queue in RQ_QUEUES.keys(): total_compute_seconds_per_queue[queue] = sum( - (p.latest_successful_run.runtime / (p.run_interval / 24)) + (p.latest_successful_run.runtime / (p.run_interval / (24 * 60))) for p in models.PipelineSchedule.objects.filter( is_active=True, run_priority=label_to_value[queue] ) diff --git a/vulnerabilities/templates/pipeline_dashboard.html b/vulnerabilities/templates/pipeline_dashboard.html index a49ac9858..a2ef36062 100644 --- a/vulnerabilities/templates/pipeline_dashboard.html +++ b/vulnerabilities/templates/pipeline_dashboard.html @@ -45,7 +45,7 @@
-
+

Pipeline Dashboard


@@ -126,13 +126,13 @@

Pipeline Dashboard

-
Pipeline ID
-
Active
-
Priority
-
Interval
-
Status
-
Last Run End Time
-
Next Run Start
+
Pipeline ID
+
Active
+
Priority
+
Interval
+
Status
+
Last Run End Time
+
Next Run Start
@@ -143,30 +143,30 @@

Pipeline Dashboard

-
-
{{ schedule.pipeline_id }}
-
{{ schedule.is_active|yesno:"Yes,No" }}
-
{{ schedule.get_run_priority_display|capfirst}}
-
+
+
{{ schedule.pipeline_id }}
+
{{ schedule.is_active|yesno:"Yes,No" }}
+
{{ schedule.get_run_priority_display|capfirst}}
+
{% if schedule.is_run_once %} Once {% else %} - {{ schedule.run_interval }} hour{{ schedule.run_interval|pluralize }} + {{ schedule.run_interval|humanize_interval }} {% endif %}
-
+
{% include "includes/job_status.html" with status=schedule.status %}
-
+
{% if schedule.latest_run_end_date %} {{ schedule.latest_run_end_date|date:"Y-m-d h:i a T" }} {% else %} N/A {% endif %}
-
+
{% if schedule.next_run_date %} {{ schedule.next_run_date|date:"Y-m-d" }} {% else %} diff --git a/vulnerabilities/templates/pipeline_run_details.html b/vulnerabilities/templates/pipeline_run_details.html index 7b602296a..94730308c 100644 --- a/vulnerabilities/templates/pipeline_run_details.html +++ b/vulnerabilities/templates/pipeline_run_details.html @@ -36,7 +36,7 @@ {% block content %}
-
+
Back to All Runs diff --git a/vulnerabilities/templates/pipeline_run_list.html b/vulnerabilities/templates/pipeline_run_list.html index cfd4c3714..c9e109b64 100644 --- a/vulnerabilities/templates/pipeline_run_list.html +++ b/vulnerabilities/templates/pipeline_run_list.html @@ -33,7 +33,7 @@
-
+
Back to Dashboard

{{ pipeline_name }} Runs

diff --git a/vulnerabilities/templatetags/utils.py b/vulnerabilities/templatetags/utils.py index 543191283..fcd611556 100644 --- a/vulnerabilities/templatetags/utils.py +++ b/vulnerabilities/templatetags/utils.py @@ -28,6 +28,11 @@ def humanize_duration(duration): return humanize_time(seconds=duration) +@register.filter +def humanize_minutes(duration): + return humanize_time(seconds=duration * 60) + + @register.simple_tag(takes_context=True) def active_item(context, url_name): """Return is-active if navbar item is active.""" @@ -64,3 +69,17 @@ def normalize_links(value): markdown_links = re.compile(r"\[([^\]]+)\]\((https?://[^\s)]+)\s*\)") return markdown_links.sub(r"\1 \2", value) + + +@register.filter +def humanize_interval(minutes): + """Humanize pipeline run interval.""" + if minutes < 60: + unit = "minute" if minutes == 1 else "minutes" + return f"{minutes} {unit}" + + hours = minutes / 60 + value = int(hours) if hours.is_integer() else round(hours, 1) + + unit = "hour" if value == 1 else "hours" + return f"{value} {unit}" diff --git a/vulnerabilities/tests/test_api_v2.py b/vulnerabilities/tests/test_api_v2.py index c9ee9a0ce..c231ac0eb 100644 --- a/vulnerabilities/tests/test_api_v2.py +++ b/vulnerabilities/tests/test_api_v2.py @@ -756,7 +756,7 @@ def test_schedule_update_anon_user_not_permitted(self, mock_create_new_job): self.assertNotEqual(response.status_code, status.HTTP_200_OK) self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN) - self.assertEqual(self.schedule1.run_interval, 24) + self.assertEqual(self.schedule1.run_interval, 720) @patch("vulnerabilities.models.PipelineSchedule.create_new_job") def test_schedule_update_with_staff_token_not_permitted(self, mock_create_new_job): @@ -771,20 +771,20 @@ def test_schedule_update_with_staff_token_not_permitted(self, mock_create_new_jo self.assertNotEqual(response.status_code, status.HTTP_200_OK) self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN) - self.assertEqual(self.schedule1.run_interval, 24) + self.assertEqual(self.schedule1.run_interval, 720) @patch("vulnerabilities.models.PipelineSchedule.create_new_job") def test_schedule_update_with_staff_session_permitted(self, mock_create_new_job): mock_create_new_job.return_value = "work-id7" self.client.login(username="admin_with_session", password="adminpassword") - data = {"run_interval": 2} + data = {"run_interval": 5} response = self.client.patch("/api/v2/pipelines/test_pipeline/", data, format="json") self.schedule1.refresh_from_db() self.assertEqual(response.status_code, status.HTTP_200_OK) self.assertNotEqual(response.status_code, status.HTTP_403_FORBIDDEN) - self.assertEqual(self.schedule1.run_interval, 2) + self.assertEqual(self.schedule1.run_interval, 5) class CodeFixV2APITest(APITestCase):