Statistics creation at each pipeline execution

#!/usr/bin/env python3

import os
import gitlab
from tabulate import tabulate

gt = gitlab.Gitlab(url=os.getenv("CI_SERVER_URL"), private_token=os.getenv("PIPELINE_STATS_TOKEN"), api_version="4")

prj = gt.projects.get(os.getenv("CI_PROJECT_PATH"))

pipeline_id=os.getenv("CI_PIPELINE_ID")
pipeline = prj.pipelines.get(pipeline_id)

def pptime(seconds):
    if seconds >= 60 * 60:
        hours = seconds / (60 * 60)
        minutes = (seconds % (60 * 60)) / 60
        return f"{hours:.0f}h {minutes:.0f}min"
    elif seconds >= 60:
        minutes = seconds / 60
        rest = seconds % 60
        return f"{minutes:.0f}min {rest:.0f}s"
    else:
        return f"{seconds:.0f}s"

def ppsize(size, decimal_places=2):
    for unit in ['B', 'KiB', 'MiB', 'GiB', 'TiB', 'PiB']:
        if size < 1024.0 or unit == 'PiB':
            break
        size /= 1024.0
    return f"{size:.{decimal_places}f} {unit}"

res=[]
total_time=0.0
total_size=0
total_log_size=0

for j in pipeline.jobs.list(iterator=True):
    if j.duration is None:
        continue # non-finished job, eg bench or pipeline stats
    size=0
    log_size=0
    if 'artifacts' in j.attributes:
        for art in j.attributes['artifacts']:
            if art['file_type'] == 'trace':
                log_size += art['size']
            else:
                size += art['size']
    res += [[j.name, j.duration, size, log_size, j.id]]
    total_time += j.duration
    total_size += size
    total_log_size += log_size

res += [['total', total_time, total_size, total_log_size, pipeline_id]]

def sortkey(v):
    return v[1]

ppres = [ [v[0], pptime(v[1]), ppsize(v[2]), ppsize(v[3]), v[4]]
          for v in sorted(res, key=sortkey) ]

print(tabulate(ppres, headers=['name', 'duration', 'artifacts size', 'log size', 'id'], tablefmt='orgtbl'))

pipeline-stats:
  image: ??? # some image with python's gitlab and tabulate libraries
  stage: .post
  dependencies: [] # no need to download artifacts
  script:
  - ./pipeline-stats.py
  when: always

| name                   | duration   | artifacts size   | log size   |      id |
|------------------------+------------+------------------+------------+---------|
| pipeline-stats         | 13s        | 0.00 B           | 0.00 B     | 5080046 |
| plugin:plugin-tutorial | 1min 25s   | 0.00 B           | 43.30 KiB  | 5076754 |
| lint                   | 2min 57s   | 0.00 B           | 40.77 KiB  | 5076753 |
| doc:ml-api:odoc        | 2min 10s   | 5.83 MiB         | 521.21 KiB | 5076758 |
| doc:init               | 2min 18s   | 1.73 MiB         | 57.51 KiB  | 5076757 |
| build:base:dev         | 2min 24s   | 234.99 MiB       | 35.75 KiB  | 5076752 |
| doc:refman             | 5min 53s   | 12.38 MiB        | 0.00 B     | 5076755 |
| docker-boot            | 5min 54s   | 0.00 B           | 7.00 KiB   | 5079931 |
| test-suite:base:dev    | 5min 2s    | 0.00 B           | 0.00 B     | 5076759 |
| doc:refman-pdf         | 5min 22s   | 11.74 MiB        | 0.00 B     | 5076756 |
| total                  | 31min 39s  | 266.67 MiB       | 705.55 KiB | 1086048 |