From efd81e1adc8eefcf760b8646bbdf1c2ae834078e Mon Sep 17 00:00:00 2001 From: Plamen Valentinov Kolev <41479552+pvk-developer@users.noreply.github.com> Date: Mon, 20 Apr 2026 09:57:16 +0200 Subject: [PATCH 1/4] Change environment from monthly to weekly --- .github/workflows/monthly_collection.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/monthly_collection.yaml b/.github/workflows/monthly_collection.yaml index 72a5693..56b2457 100644 --- a/.github/workflows/monthly_collection.yaml +++ b/.github/workflows/monthly_collection.yaml @@ -8,7 +8,7 @@ on: jobs: monthly_github_collection: - environment: monthly + environment: weekly runs-on: ubuntu-latest steps: - uses: actions/checkout@v6 From 2067b0160ae387a784bfd447f8091d48f55d03b8 Mon Sep 17 00:00:00 2001 From: Plamen Valentinov Kolev Date: Mon, 20 Apr 2026 15:08:20 +0200 Subject: [PATCH 2/4] Update consolidate upload path --- gitmetrics/consolidate.py | 16 ++++++++++++++++ gitmetrics/drive.py | 2 +- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/gitmetrics/consolidate.py b/gitmetrics/consolidate.py index f0998ff..7bcaae3 100644 --- a/gitmetrics/consolidate.py +++ b/gitmetrics/consolidate.py @@ -12,6 +12,7 @@ METRICS_SHEET_NAME, VALUE_COLUMN_NAME, ) +from gitmetrics.drive import _get_drive_client, is_drive_path, split_drive_path from gitmetrics.output import create_spreadsheet, load_spreadsheet OUTPUT_FILENAME = 'gitmetrics_consolidated_summary_to_date' @@ -62,6 +63,21 @@ def consolidate_metrics(projects, output_folder, dry_run=False, verbose=True): if verbose: LOGGER.info(f'Sheet Name: {SHEET_NAME}') LOGGER.info(consolidated_df.to_string()) + if not dry_run: output_path = os.path.join(output_folder, OUTPUT_FILENAME) create_spreadsheet(output_path=output_path, sheets=sheets) + + if is_drive_path(output_folder): + drive = _get_drive_client() + gdrive_folder = output_folder.rstrip('/') + '/' + folder_id, _ = split_drive_path(gdrive_folder) + + folder = drive.CreateFile({'id': folder_id}) + folder.FetchMetadata(fields='parents') + + parents = folder.get('parents') or [] + parent_id = parents[0].get('id') + + output_path = f'gdrive://{parent_id}/{OUTPUT_FILENAME}' + create_spreadsheet(output_path=output_path, sheets=sheets) diff --git a/gitmetrics/drive.py b/gitmetrics/drive.py index 210070f..5094936 100644 --- a/gitmetrics/drive.py +++ b/gitmetrics/drive.py @@ -98,7 +98,7 @@ def upload_spreadsheet(content, filename, folder): retries = 0 while retries != MAX_UPLOAD_RETRIES: try: - if drive_file['mimeType'] == SPREADSHEET_MIMETYPE: + if drive_file.get('mimeType') == SPREADSHEET_MIMETYPE: drive_file.Upload() else: drive_file.Upload({'convert': True}) From 68e7592d79a4ee42fa78c61fe7145b55ab37599c Mon Sep 17 00:00:00 2001 From: Plamen Valentinov Kolev Date: Mon, 20 Apr 2026 15:58:57 +0200 Subject: [PATCH 3/4] Move PyTorchLightning to weekly --- monthly_extraction_config.yaml | 1 - weekly_extraction_config.yaml | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/monthly_extraction_config.yaml b/monthly_extraction_config.yaml index 31c3f75..47d6803 100644 --- a/monthly_extraction_config.yaml +++ b/monthly_extraction_config.yaml @@ -8,4 +8,3 @@ projects: scikit-learn: ray: airbyte: - PyTorchLightning: diff --git a/weekly_extraction_config.yaml b/weekly_extraction_config.yaml index e3fb698..077a22d 100644 --- a/weekly_extraction_config.yaml +++ b/weekly_extraction_config.yaml @@ -11,3 +11,4 @@ projects: pycaret: snorkel: spacy: + PyTorchLightning: From 376758e446fc927781ecbf37dffd4f6fbfd90abb Mon Sep 17 00:00:00 2001 From: Plamen Valentinov Kolev Date: Mon, 20 Apr 2026 16:19:15 +0200 Subject: [PATCH 4/4] Make the upload of the consolidation be at the top level --- .github/workflows/monthly_collection.yaml | 2 +- .github/workflows/weekly_collection.yaml | 2 +- gitmetrics/__main__.py | 13 ++++++++----- gitmetrics/consolidate.py | 5 ++++- 4 files changed, 14 insertions(+), 8 deletions(-) diff --git a/.github/workflows/monthly_collection.yaml b/.github/workflows/monthly_collection.yaml index 56b2457..3c18314 100644 --- a/.github/workflows/monthly_collection.yaml +++ b/.github/workflows/monthly_collection.yaml @@ -36,7 +36,7 @@ jobs: - name: Consolidate GitHub Data run: | uv run gitmetrics consolidate \ - --config-file monthly_extraction_config.yaml \ + --config-file weekly_extraction_config.yaml monthly_extraction_config.yaml \ --output-folder ${{ secrets.OUTPUT_FOLDER }} env: PYDRIVE_CREDENTIALS: ${{ secrets.PYDRIVE_CREDENTIALS }} diff --git a/.github/workflows/weekly_collection.yaml b/.github/workflows/weekly_collection.yaml index 3b71bec..a8b3ac7 100644 --- a/.github/workflows/weekly_collection.yaml +++ b/.github/workflows/weekly_collection.yaml @@ -32,7 +32,7 @@ jobs: - name: Consolidate GitHub Data run: | uv run gitmetrics consolidate \ - --config-file weekly_extraction_config.yaml \ + --config-file weekly_extraction_config.yaml monthly_extraction_config.yaml \ --output-folder ${{ secrets.OUTPUT_FOLDER }} env: PYDRIVE_CREDENTIALS: ${{ secrets.PYDRIVE_CREDENTIALS }} diff --git a/gitmetrics/__main__.py b/gitmetrics/__main__.py index bd073a2..72c6f39 100644 --- a/gitmetrics/__main__.py +++ b/gitmetrics/__main__.py @@ -140,11 +140,13 @@ def _summarize(args, parser): def _consolidate(args, parser): - config = _load_config(args.config_file) - projects = config['projects'] + all_projects = [] + for config_file in args.config_file: + config = _load_config(config_file) + all_projects.extend(config.get('projects', [])) consolidate_metrics( - projects=projects, + projects=all_projects, output_folder=args.output_folder, dry_run=args.dry_run, verbose=args.verbose, @@ -223,8 +225,9 @@ def _get_parser(): '-c', '--config-file', type=str, - default=DEFAULT_PROJECT_DEFINITIONS, - help='Path to the configuration file.', + nargs='+', + default=[DEFAULT_PROJECT_DEFINITIONS], + help='Path(s) to configuration file(s).', ) consolidate.add_argument( '-d', diff --git a/gitmetrics/consolidate.py b/gitmetrics/consolidate.py index 7bcaae3..666999d 100644 --- a/gitmetrics/consolidate.py +++ b/gitmetrics/consolidate.py @@ -56,6 +56,7 @@ def consolidate_metrics(projects, output_folder, dry_run=False, verbose=True): row_info.update(row_values) if verbose: LOGGER.info(f' {project} values: {row_info}') + rows.append(row_info) consolidated_df = pd.DataFrame(rows) @@ -66,7 +67,6 @@ def consolidate_metrics(projects, output_folder, dry_run=False, verbose=True): if not dry_run: output_path = os.path.join(output_folder, OUTPUT_FILENAME) - create_spreadsheet(output_path=output_path, sheets=sheets) if is_drive_path(output_folder): drive = _get_drive_client() @@ -81,3 +81,6 @@ def consolidate_metrics(projects, output_folder, dry_run=False, verbose=True): output_path = f'gdrive://{parent_id}/{OUTPUT_FILENAME}' create_spreadsheet(output_path=output_path, sheets=sheets) + + else: + create_spreadsheet(output_path=output_path, sheets=sheets)