From a496dc28e36bf4a985675de2563b48062a7d089c Mon Sep 17 00:00:00 2001 From: Daniel Schmidt Date: Tue, 2 Sep 2025 16:29:51 -0700 Subject: [PATCH 1/4] WIP: Pruning script --- .dockerignore | 1 + .gitignore | 2 ++ Gemfile | 4 ++++ Gemfile.lock | 36 ++++++++++++++++++++++++++++++- app.rb | 53 +++++++++++++++++++++++++++++++++++++++++++++- docker-compose.yml | 6 ++++++ 6 files changed, 100 insertions(+), 2 deletions(-) diff --git a/.dockerignore b/.dockerignore index 2c72a5f..17fc17c 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,5 +1,6 @@ .dockerignore .DS_Store +.env .git* .gitignore docker-compose* diff --git a/.gitignore b/.gitignore index fd5106f..0baf78e 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,3 @@ .DS_STORE +.env +app.rb diff --git a/Gemfile b/Gemfile index 116cd7f..4d7b858 100644 --- a/Gemfile +++ b/Gemfile @@ -2,8 +2,12 @@ source "https://rubygems.org" +gem "octokit", "~> 10.0" gem "puma" gem "rack-test" gem "rackup" gem "rspec" +gem "sinatra-contrib", "~> 4.1" gem "sinatra" + +gem "pry", "~> 0.15.2", :group => :development diff --git a/Gemfile.lock b/Gemfile.lock index 7506843..a631956 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,12 +1,33 @@ GEM remote: https://rubygems.org/ specs: + addressable (2.8.7) + public_suffix (>= 2.0.2, < 7.0) base64 (0.3.0) + coderay (1.1.3) diff-lcs (1.6.2) + faraday (2.13.4) + faraday-net_http (>= 2.0, < 3.5) + json + logger + faraday-net_http (3.4.1) + net-http (>= 0.5.0) + json (2.13.2) logger (1.7.0) + method_source (1.1.0) + multi_json (1.17.0) mustermann (3.0.4) ruby2_keywords (~> 0.0.1) + net-http (0.6.0) + uri nio4r (2.7.4) + octokit (10.0.0) + faraday (>= 1, < 3) + sawyer (~> 0.9) + pry (0.15.2) + coderay (~> 1.1) + method_source (~> 1.0) + public_suffix (6.0.2) puma (6.6.1) nio4r (~> 2.0) rack (3.2.0) @@ -35,6 +56,9 @@ GEM rspec-support (~> 3.13.0) rspec-support (3.13.5) ruby2_keywords (0.0.5) + sawyer (0.9.2) + addressable (>= 2.3.5) + faraday (>= 0.17.3, < 3) sinatra (4.1.1) logger (>= 1.6.0) mustermann (~> 3.0) @@ -42,18 +66,28 @@ GEM rack-protection (= 4.1.1) rack-session (>= 2.0.0, < 3) tilt (~> 2.0) + sinatra-contrib (4.1.1) + multi_json (>= 0.0.2) + mustermann (~> 3.0) + rack-protection (= 4.1.1) + sinatra (= 4.1.1) + tilt (~> 2.0) tilt (2.6.1) + uri (1.0.3) PLATFORMS aarch64-linux ruby DEPENDENCIES + octokit (~> 10.0) + pry (~> 0.15.2) puma rack-test rackup rspec sinatra + sinatra-contrib (~> 4.1) BUNDLED WITH - 2.6.9 + 2.6.5 diff --git a/app.rb b/app.rb index 0651bdf..8dc759c 100644 --- a/app.rb +++ b/app.rb @@ -1,5 +1,56 @@ +require 'date' +require 'octokit' require 'sinatra' +require 'sinatra/json' + +configure :development do + set :logging, Logger::DEBUG + set :server_settings, timeout: 60 +end + +def has_permatag?(version, permatags) + (version['metadata']['container']['tags'] & permatags).any? +end + +def younger_than?(version, days_old) + cutoff = Time.now - 60*60*24*days_old + version['created_at'] > cutoff +end + +github = Octokit::Client.new(per_page: 100, auto_paginate: true) get '/' do - "Hello, world!" + 'Hello, world!' +end + +get '/prunables' do + packages = github.get('orgs/BerkeleyLibrary/packages', {package_type: :container}) + logger.info "Scanning #{packages.size} packages for prunable images: #{packages.collect(&:name).sort}" + + packages.each do |pkg| + logger.info "Determining prunable images for #{pkg.name}" + + next unless pkg.repository + + permatags = %w(latest edge) + permatags += github.branches(pkg.repository.full_name).collect(&:name) + permatags += github.tags(pkg.repository.full_name).collect(&:name) + + images = github.get("orgs/#{pkg.owner.login}/packages/#{pkg.package_type}/#{pkg.name}/versions").collect do |image| + if has_permatag? image, permatags + verdict = :permatagged + elsif younger_than? image, 7 + verdict = :recent + else + verdict = :prunable + end + # { image:, verdict: } + end + + json({ + repo: pkg.repository.full_name, + package: pkg.name, + images:, + }) + end end diff --git a/docker-compose.yml b/docker-compose.yml index c504ee5..aa1016d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -3,7 +3,13 @@ services: app: build: . + env_file: + - .env ports: - 4567:4567 volumes: - ./:/opt/app + develop: + watch: + - action: rebuild + path: . From daba27172e5fbf0400afa2e4b6870ca32086ad55 Mon Sep 17 00:00:00 2001 From: Daniel Schmidt Date: Tue, 2 Sep 2025 16:47:05 -0700 Subject: [PATCH 2/4] gets status working --- app.rb | 52 ++++++++++++++++++++++++++++------------------------ 1 file changed, 28 insertions(+), 24 deletions(-) diff --git a/app.rb b/app.rb index 8dc759c..c850917 100644 --- a/app.rb +++ b/app.rb @@ -23,34 +23,38 @@ def younger_than?(version, days_old) 'Hello, world!' end -get '/prunables' do +# Returns all container packages along with their pruning status, i.e. whether they can/can't be pruned and why +get '/images' do packages = github.get('orgs/BerkeleyLibrary/packages', {package_type: :container}) logger.info "Scanning #{packages.size} packages for prunable images: #{packages.collect(&:name).sort}" - packages.each do |pkg| - logger.info "Determining prunable images for #{pkg.name}" - - next unless pkg.repository - - permatags = %w(latest edge) - permatags += github.branches(pkg.repository.full_name).collect(&:name) - permatags += github.tags(pkg.repository.full_name).collect(&:name) - - images = github.get("orgs/#{pkg.owner.login}/packages/#{pkg.package_type}/#{pkg.name}/versions").collect do |image| - if has_permatag? image, permatags - verdict = :permatagged - elsif younger_than? image, 7 - verdict = :recent - else - verdict = :prunable + prunables = [].tap do |sofar| + packages.each do |pkg| + logger.info "Determining prunable images for #{pkg.name}" + + next unless pkg.repository + + permatags = %w(latest edge) + permatags += github.branches(pkg.repository.full_name).collect(&:name) + permatags += github.tags(pkg.repository.full_name).collect(&:name) + + github.get("orgs/#{pkg.owner.login}/packages/#{pkg.package_type}/#{pkg.name}/versions").each do |image| + if has_permatag? image, permatags + pruning_status = :permatagged + elsif younger_than? image, 7 + pruning_status = :recent + else + pruning_status = :prunable + end + + sofar << { + image: image.to_attrs, + pruning_status: pruning_status, + can_be_pruned: pruning_status == :prunable, + } end - # { image:, verdict: } end - - json({ - repo: pkg.repository.full_name, - package: pkg.name, - images:, - }) end + + json prunables end From df2b0caf8f94eedf35b79d3163f4bac1f5fbbcb5 Mon Sep 17 00:00:00 2001 From: Daniel Schmidt Date: Wed, 3 Sep 2025 09:21:34 -0700 Subject: [PATCH 3/4] removes .env from ci file --- docker-compose.ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/docker-compose.ci.yml b/docker-compose.ci.yml index 9f04aa0..3305ff2 100644 --- a/docker-compose.ci.yml +++ b/docker-compose.ci.yml @@ -4,6 +4,7 @@ services: app: build: !reset image: ${DOCKER_APP_IMAGE} + env_file: !reset ports: !reset volumes: !override - $ARTIFACTS_DIR:/tmp/artifacts From 049b5725e29bfc5b6aaa7093c966f1a068f8773e Mon Sep 17 00:00:00 2001 From: Daniel Schmidt Date: Mon, 8 Sep 2025 17:04:05 -0700 Subject: [PATCH 4/4] adds pagination --- Gemfile | 5 ++- Gemfile.lock | 5 +++ app.rb | 122 +++++++++++++++++++++++++++++++++++++-------------- 3 files changed, 99 insertions(+), 33 deletions(-) diff --git a/Gemfile b/Gemfile index 4d7b858..ff273fa 100644 --- a/Gemfile +++ b/Gemfile @@ -10,4 +10,7 @@ gem "rspec" gem "sinatra-contrib", "~> 4.1" gem "sinatra" -gem "pry", "~> 0.15.2", :group => :development +group :development do + gem "pry", "~> 0.15.2" + gem "pry-doc", "~> 1.6" +end diff --git a/Gemfile.lock b/Gemfile.lock index a631956..51128dc 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -27,6 +27,9 @@ GEM pry (0.15.2) coderay (~> 1.1) method_source (~> 1.0) + pry-doc (1.6.0) + pry (~> 0.11) + yard (~> 0.9.11) public_suffix (6.0.2) puma (6.6.1) nio4r (~> 2.0) @@ -74,6 +77,7 @@ GEM tilt (~> 2.0) tilt (2.6.1) uri (1.0.3) + yard (0.9.37) PLATFORMS aarch64-linux @@ -82,6 +86,7 @@ PLATFORMS DEPENDENCIES octokit (~> 10.0) pry (~> 0.15.2) + pry-doc (~> 1.6) puma rack-test rackup diff --git a/app.rb b/app.rb index c850917..032df73 100644 --- a/app.rb +++ b/app.rb @@ -1,5 +1,6 @@ require 'date' require 'octokit' +require 'faraday' require 'sinatra' require 'sinatra/json' @@ -8,53 +9,110 @@ set :server_settings, timeout: 60 end -def has_permatag?(version, permatags) - (version['metadata']['container']['tags'] & permatags).any? +stack = Faraday::RackBuilder.new do |builder| + builder.use Octokit::Middleware::FollowRedirects + builder.use Octokit::Response::RaiseError + builder.use Octokit::Response::FeedParser + builder.response :logger, nil, { headers: true, bodies: true, errors: true } do |logger| + logger.filter(/(Authorization: "(token|Bearer) )(\w+)/, '\1[REMOVED]') + end + builder.adapter Faraday.default_adapter end +Octokit.middleware = stack + +module Logging + def logger + Logging.logger + end -def younger_than?(version, days_old) - cutoff = Time.now - 60*60*24*days_old - version['created_at'] > cutoff + def self.logger + @logger ||= Logger.new(STDOUT) + end end -github = Octokit::Client.new(per_page: 100, auto_paginate: true) +class RegistryPruner + include Logging -get '/' do - 'Hello, world!' -end + attr_reader :github, :org -# Returns all container packages along with their pruning status, i.e. whether they can/can't be pruned and why -get '/images' do - packages = github.get('orgs/BerkeleyLibrary/packages', {package_type: :container}) - logger.info "Scanning #{packages.size} packages for prunable images: #{packages.collect(&:name).sort}" + def initialize(github: nil, org: 'BerkeleyLibrary') + @github = github || Octokit::Client.new(per_page: 100, auto_paginate: true) + @org = org + @inventory = nil + end - prunables = [].tap do |sofar| - packages.each do |pkg| - logger.info "Determining prunable images for #{pkg.name}" + def inventory + @inventory ||= [].then { refresh_inventory! } + end - next unless pkg.repository + def prune!(days_old = 7) + cutoff = Time.now - (60*60*24 * days_old) - permatags = %w(latest edge) - permatags += github.branches(pkg.repository.full_name).collect(&:name) - permatags += github.tags(pkg.repository.full_name).collect(&:name) + inventory.each do |image| + if image[:created_at] > cutoff + logger.debug "SKIPPING: Image #{image[:package]}/#{image[:version]} created recently: #{image[:created_at]}" + next + end + + permatags = image[:tags] & image[:repo_permatags] + if permatags.any? + logger.debug "SKIPPING: Image #{image[:package]}/#{image[:version]} has permatags: #{permatags.sort.join(', ')}" + next + end - github.get("orgs/#{pkg.owner.login}/packages/#{pkg.package_type}/#{pkg.name}/versions").each do |image| - if has_permatag? image, permatags - pruning_status = :permatagged - elsif younger_than? image, 7 - pruning_status = :recent + begin + logger.debug("Deleting image: #{image[:url]}") + github.delete image[:url], nil + rescue Octokit::BadRequest => e + logger.error(e) + if e.message =~ /cannot be deleted/ + next else - pruning_status = :prunable + raise end + end + end + end - sofar << { - image: image.to_attrs, - pruning_status: pruning_status, - can_be_pruned: pruning_status == :prunable, - } + def refresh_inventory! + @inventory = [].tap do |images| + github.get("orgs/#{org}/packages", { package_type: :container }).each do |pkg| + next unless pkg.repository + + repo = pkg.repository.full_name + repo_permatags = permatags_for(pkg) + next_page = "orgs/#{org}/packages/container/#{pkg.name}/versions" + + loop do + github.get(next_page).each do |image| + images << { + url: "orgs/#{org}/packages/container/#{pkg.name}/versions/#{image.id}", + package: pkg.name, + version: image.id, + created_at: image['created_at'], + tags: image['metadata']['container']['tags'], + repo:, + repo_permatags:, + } + end + next_page = github.last_response.rels[:next]&.href + break if next_page.nil? + end end end end - json prunables + def permatags_for(pkg) + %w(latest edge).tap do |permatags| + permatags.concat github.branches(pkg.repository.full_name).collect(&:name) + permatags.concat github.tags(pkg.repository.full_name).collect(&:name) + permatags.sort! + end + end +end + +get '/images' do + pruner = RegistryPruner.new + inventory = pruner.inventory + json({ inventory: }) end