diff --git a/.dockerignore b/.dockerignore index 2c72a5f..17fc17c 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,5 +1,6 @@ .dockerignore .DS_Store +.env .git* .gitignore docker-compose* diff --git a/.gitignore b/.gitignore index fd5106f..0baf78e 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,3 @@ .DS_STORE +.env +app.rb diff --git a/Gemfile b/Gemfile index 116cd7f..ff273fa 100644 --- a/Gemfile +++ b/Gemfile @@ -2,8 +2,15 @@ source "https://rubygems.org" +gem "octokit", "~> 10.0" gem "puma" gem "rack-test" gem "rackup" gem "rspec" +gem "sinatra-contrib", "~> 4.1" gem "sinatra" + +group :development do + gem "pry", "~> 0.15.2" + gem "pry-doc", "~> 1.6" +end diff --git a/Gemfile.lock b/Gemfile.lock index 7506843..51128dc 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,12 +1,36 @@ GEM remote: https://rubygems.org/ specs: + addressable (2.8.7) + public_suffix (>= 2.0.2, < 7.0) base64 (0.3.0) + coderay (1.1.3) diff-lcs (1.6.2) + faraday (2.13.4) + faraday-net_http (>= 2.0, < 3.5) + json + logger + faraday-net_http (3.4.1) + net-http (>= 0.5.0) + json (2.13.2) logger (1.7.0) + method_source (1.1.0) + multi_json (1.17.0) mustermann (3.0.4) ruby2_keywords (~> 0.0.1) + net-http (0.6.0) + uri nio4r (2.7.4) + octokit (10.0.0) + faraday (>= 1, < 3) + sawyer (~> 0.9) + pry (0.15.2) + coderay (~> 1.1) + method_source (~> 1.0) + pry-doc (1.6.0) + pry (~> 0.11) + yard (~> 0.9.11) + public_suffix (6.0.2) puma (6.6.1) nio4r (~> 2.0) rack (3.2.0) @@ -35,6 +59,9 @@ GEM rspec-support (~> 3.13.0) rspec-support (3.13.5) ruby2_keywords (0.0.5) + sawyer (0.9.2) + addressable (>= 2.3.5) + faraday (>= 0.17.3, < 3) sinatra (4.1.1) logger (>= 1.6.0) mustermann (~> 3.0) @@ -42,18 +69,30 @@ GEM rack-protection (= 4.1.1) rack-session (>= 2.0.0, < 3) tilt (~> 2.0) + sinatra-contrib (4.1.1) + multi_json (>= 0.0.2) + mustermann (~> 3.0) + rack-protection (= 4.1.1) + sinatra (= 4.1.1) + tilt (~> 2.0) tilt (2.6.1) + uri (1.0.3) + yard (0.9.37) PLATFORMS aarch64-linux ruby DEPENDENCIES + octokit (~> 10.0) + pry (~> 0.15.2) + pry-doc (~> 1.6) puma rack-test rackup rspec sinatra + sinatra-contrib (~> 4.1) BUNDLED WITH - 2.6.9 + 2.6.5 diff --git a/app.rb b/app.rb index 0651bdf..032df73 100644 --- a/app.rb +++ b/app.rb @@ -1,5 +1,118 @@ +require 'date' +require 'octokit' +require 'faraday' require 'sinatra' +require 'sinatra/json' -get '/' do - "Hello, world!" +configure :development do + set :logging, Logger::DEBUG + set :server_settings, timeout: 60 +end + +stack = Faraday::RackBuilder.new do |builder| + builder.use Octokit::Middleware::FollowRedirects + builder.use Octokit::Response::RaiseError + builder.use Octokit::Response::FeedParser + builder.response :logger, nil, { headers: true, bodies: true, errors: true } do |logger| + logger.filter(/(Authorization: "(token|Bearer) )(\w+)/, '\1[REMOVED]') + end + builder.adapter Faraday.default_adapter +end +Octokit.middleware = stack + +module Logging + def logger + Logging.logger + end + + def self.logger + @logger ||= Logger.new(STDOUT) + end +end + +class RegistryPruner + include Logging + + attr_reader :github, :org + + def initialize(github: nil, org: 'BerkeleyLibrary') + @github = github || Octokit::Client.new(per_page: 100, auto_paginate: true) + @org = org + @inventory = nil + end + + def inventory + @inventory ||= [].then { refresh_inventory! } + end + + def prune!(days_old = 7) + cutoff = Time.now - (60*60*24 * days_old) + + inventory.each do |image| + if image[:created_at] > cutoff + logger.debug "SKIPPING: Image #{image[:package]}/#{image[:version]} created recently: #{image[:created_at]}" + next + end + + permatags = image[:tags] & image[:repo_permatags] + if permatags.any? + logger.debug "SKIPPING: Image #{image[:package]}/#{image[:version]} has permatags: #{permatags.sort.join(', ')}" + next + end + + begin + logger.debug("Deleting image: #{image[:url]}") + github.delete image[:url], nil + rescue Octokit::BadRequest => e + logger.error(e) + if e.message =~ /cannot be deleted/ + next + else + raise + end + end + end + end + + def refresh_inventory! + @inventory = [].tap do |images| + github.get("orgs/#{org}/packages", { package_type: :container }).each do |pkg| + next unless pkg.repository + + repo = pkg.repository.full_name + repo_permatags = permatags_for(pkg) + next_page = "orgs/#{org}/packages/container/#{pkg.name}/versions" + + loop do + github.get(next_page).each do |image| + images << { + url: "orgs/#{org}/packages/container/#{pkg.name}/versions/#{image.id}", + package: pkg.name, + version: image.id, + created_at: image['created_at'], + tags: image['metadata']['container']['tags'], + repo:, + repo_permatags:, + } + end + next_page = github.last_response.rels[:next]&.href + break if next_page.nil? + end + end + end + end + + def permatags_for(pkg) + %w(latest edge).tap do |permatags| + permatags.concat github.branches(pkg.repository.full_name).collect(&:name) + permatags.concat github.tags(pkg.repository.full_name).collect(&:name) + permatags.sort! + end + end +end + +get '/images' do + pruner = RegistryPruner.new + inventory = pruner.inventory + json({ inventory: }) end diff --git a/docker-compose.ci.yml b/docker-compose.ci.yml index 9f04aa0..3305ff2 100644 --- a/docker-compose.ci.yml +++ b/docker-compose.ci.yml @@ -4,6 +4,7 @@ services: app: build: !reset image: ${DOCKER_APP_IMAGE} + env_file: !reset ports: !reset volumes: !override - $ARTIFACTS_DIR:/tmp/artifacts diff --git a/docker-compose.yml b/docker-compose.yml index c504ee5..aa1016d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -3,7 +3,13 @@ services: app: build: . + env_file: + - .env ports: - 4567:4567 volumes: - ./:/opt/app + develop: + watch: + - action: rebuild + path: .