add autoad dockerization support

Author: ppalmes

DockerizedAutoAD/CondaPkg.toml

@@ -1,4 +1,5 @@
 [deps]
 mlflow = ""
-pycaret = ""
+pycaret = "3.3.*"
+python = "3.11.*"
 requests = ""

DockerizedAutoAD/Dockerfile

@@ -3,8 +3,8 @@ LABEL MAINTAINER='Paulito Palmes "paulpalmes@ie.ibm.com"'
 RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y locales build-essential git 
 
 RUN sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen && \
-    dpkg-reconfigure --frontend=noninteractive locales && \
-    update-locale LANG=en_US.UTF-8
+  dpkg-reconfigure --frontend=noninteractive locales && \
+  update-locale LANG=en_US.UTF-8
 
 ENV LANG en_US.UTF-8
 ENV LANGUAGE en_US:en
@@ -19,7 +19,7 @@ COPY automl_sysimage.jl /amlp/
 COPY automl_precompile.jl /amlp/
 ADD https://github.com/IBM/AutoMLPipeline.jl.git /amlp/AutoMLPipeline/
 RUN julia --project -e 'using Pkg;\
-  Pkg.develop(path="./AutoMLPipeline/AutoAI/");\
+  Pkg.develop(path="./AutoMLPipeline/AutoAD/");\
   Pkg.instantiate();\
   Pkg.update();\
   Pkg.precompile();\

DockerizedAutoAD/Project.toml

@@ -1,7 +1,7 @@
 [deps]
 AMLPipelineBase = "e3c3008a-8869-4d53-9f34-c96f99c8a2b6"
 ArgParse = "c7e460c6-2fb9-53a9-8c5b-16f535851c63"
-AutoAI = "78f36755-6713-4987-acdd-2a78ef0fc2e4"
+AutoAD = "a90a1e1f-13cb-41ec-be1f-24b06fd2b474"
 AutoMLPipeline = "08437348-eef5-4817-bc1b-d4e9459680d6"
 CSV = "336ed68f-0bac-5ca0-87d4-7b16caf5d00b"
 CondaPkg = "992eb4ea-22a4-4c89-a5bb-47a3300528ab"

DockerizedAutoAD/automl_precompile.jl

@@ -5,4 +5,4 @@ import AutoMLPipeline
 include(joinpath(pkgdir(AutoMLPipeline), "test", "runtests.jl"))
 
 import AutoAI
-include(joinpath(pkgdir(AutoAI), "test", "runtests.jl"))
+include(joinpath(pkgdir(AutoAD), "test", "runtests.jl"))

DockerizedAutoAD/automl_sysimage.jl

@@ -1,6 +1,6 @@
 using PackageCompiler
 create_sysimage(["AMLPipelineBase", "DataFrames", "StatsBase",
     "ArgParse", "AutoMLPipeline", "CSV", "Dates", "Distributed",
-    "Random", "ArgParse", "Test", "Distributed", "PythonCall","AutoAI",
+    "Random", "ArgParse", "Test", "Distributed", "PythonCall","AutoAD",
     "Statistics", "Serialization", "StatsBase", "Test"],
   sysimage_path="automl.so", precompile_execution_file="automl_precompile.jl")

DockerizedAutoAD/main.jl

@@ -1,3 +1,4 @@
+using AutoAD: AutoMLFlowAnomalyDetections
 using Distributed
 using ArgParse
 using CSV
@@ -14,108 +15,63 @@ function parse_commandline()
     arg_type = String
     default = "http://localhost:8080"
     "--prediction_type", "-t"
-    help = "classification, regression, anomalydetection"
+    help = "timeseriesprediction, anomalydetection"
     arg_type = String
-    default = "classification"
-    #default = "anomalydetection"
-    #default = "regression"
-    "--complexity", "-c"
-    help = "pipeline complexity"
-    arg_type = String
-    default = "low"
+    default = "anomalydetecion"
     "--output_file", "-o"
     help = "output location"
     arg_type = String
     default = "NONE"
-    "--nfolds", "-f"
-    help = "number of crossvalidation folds"
-    arg_type = Int64
-    default = 3
-    "--nworkers", "-w"
-    help = "number of workers"
-    arg_type = Int64
-    default = 5
     "--votepercent", "-v"
     help = "votepercent for anomalydetection ensembles"
     arg_type = Float64
     default = 0.0
-    "--no_save"
-    help = "save model"
-    action = :store_true
-    "--predict_only"
-    help = "no training, predict only"
-    action = :store_true
     "--runid"
     help = "runid of experiment for trained model"
     arg_type = String
     default = "NONE"
     "csvfile"
     help = "input csv file"
     required = true
-    #default = "iris.csv"
-    #default = "iris_reg.csv"
   end
   return parse_args(s; as_symbols=true)
 end
 
-#const _cliargs = (; parse_commandline()...)
-_cliargs = parse_commandline()
-#const _workers = _cliargs[:workers]
-_workers = _cliargs[:nworkers]
+const _cliargs = (; parse_commandline()...)
+#_cliargs = parse_commandline()
+const _workers = _cliargs[:workers]
+#_workers = _cliargs[:nworkers]
 
 nprocs() == 1 && addprocs(_workers; exeflags=["--project=$(Base.active_project())"])
 
-@everywhere using AutoAI
-
-function autoclassmode(args::Dict)
-  url = args[:url]
-  complexity = args[:complexity]
-  nfolds = args[:nfolds]
-  nworkers = args[:nworkers]
-  prediction_type = args[:prediction_type]
-  impl_args = (; complexity, nfolds, nworkers, prediction_type) |> pairs |> Dict
-  fname = _cliargs[:csvfile]
-  df = CSV.read(fname, DataFrame)
-  X = df[:, 1:end-1]
-  Y = df[:, end] |> collect
-  autoclass = AutoMLFlowClassification(Dict(:url => url, :impl_args => impl_args))
-  Yc = fit_transform!(autoclass, X, Y)
-  println("accuracy = ", mean(Y .== Yc))
-end
+@everywhere using AutoAD
 
-function autoregmode(args::Dict)
+function autoadmode(args::Dict)
   url = args[:url]
-  complexity = args[:complexity]
-  nfolds = args[:nfolds]
-  nworkers = args[:nworkers]
   prediction_type = args[:prediction_type]
-  impl_args = (; complexity, nfolds, nworkers, prediction_type) |> pairs |> Dict
-  fname = _cliargs[:csvfile]
+  votepercent = args[:votepercent]
+  dictargs = (; votepercent, prediction_type) |> pairs |> Dict
+  fname = args[:csvfile]
   df = CSV.read(fname, DataFrame)
-  X = df[:, 1:end-1]
-  Y = df[:, end] |> collect
-  autoreg = AutoMLFlowRegression(Dict(:url => url, :impl_args => impl_args))
-  Yc = fit_transform!(autoreg, X, Y)
-  println("mse = ", mean((Y - Yc) .^ 2))
+  X = df[:, 1:1]
+  autoad = AutoMLFlowAnomalyDetections(Dict(:url => url, dictargs...))
+  Yc = fit_transform!(autoad, X)
+  println("output:", Yc |> x -> first(x, 5))
+  return Yc
 end
 
-function autoadmode(args::Dict)
-  url = args[:url]
-  votepercent = args[:votepercent]
-  X = vcat(5 * cos.(-10:10), sin.(-30:30), 3 * cos.(-10:10), 2 * tan.(-10:10), sin.(-30:30)) |> x -> DataFrame([x], :auto)
-  mlfad = AutoMLFlowAnomalyDetection(Dict(:url => url, :impl_args => Dict(:votepercent => votepercent)))
-  Yc = fit_transform!(mlfad, X)
-  println(Yc |> x -> first(x, 5))
+function autotsmode(args::Dict)
 end
 
-function main()
-  predtype = _cliargs[:prediction_type]
-  if predtype == "classification"
-    autoclassmode(_cliargs)
-  elseif predtype == "regression"
-    autoregmode(_cliargs)
+function main(args)
+  predtype = args[:prediction_type]
+  if predtype == "anomalydetection"
+    autoadmode(args)
+  elseif predtype == "timeseriesprediction"
+    autotsmode(_cliargs)
   elseif predtype == "anomalydetection"
     autoadmode(_cliargs)
   end
 end
-main()
+
+main(_cliargs)

DockerizedAutoAD/run.sh

@@ -1,5 +1,5 @@
-docker build -t automlai --platform=linux/amd64 .
-docker run -it --rm --platform=linux/amd64 automlai
+docker build -t automlad --platform=linux/amd64 .
+docker run -it --rm --platform=linux/amd64 automlad
 
 # julia --project -- ./main.jl -c high -t regression -f 3 -w 7 iris_reg.csv
 # julia --project -- ./main.jl -c low -t classification -f 3 -w 3 iris.csv