cyclestreets · oliverlockwood · Oct 8, 2018 · Oct 11, 2018 · Oct 12, 2018 · May 1, 2020
diff --git a/README.md b/README.md
@@ -1,5 +1,14 @@
 [![Build Status](https://travis-ci.com/cyclestreets/android.svg?branch=master)](https://travis-ci.com/cyclestreets/android)
 
+# Design notes for offline maps
+
+###  Settings -> Maps display
+Replace "Vector mapfile" with "Offline maps"
+This can be a new Activity (like e.g. LocationsActivity)
+Contains a list of available map packs and their sizes, with last-updated 
+Need a simple script to parse  
+
+
 # Cyclestreets Android App
 
 ## What is it?

diff --git a/apacheIndexPage b/apacheIndexPage
@@ -0,0 +1,67 @@
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 3.2 Final//EN">
+<html>
+ <head>
+  <title>Index of /pub/Mirrors/download.mapsforge.org/maps/v4/europe</title>
+ </head>
+ <body>
+<h1>Index of /pub/Mirrors/download.mapsforge.org/maps/v4/europe</h1>
+  <table>
+   <tr><th valign="top"><img src="/icons/blank.gif" alt="[ICO]"></th><th><a href="?C=N;O=D">Name</a></th><th><a href="?C=M;O=A">Last modified</a></th><th><a href="?C=S;O=A">Size</a></th><th><a href="?C=D;O=A">Description</a></th></tr>
+   <tr><th colspan="5"><hr></th></tr>
+<tr><td valign="top"><img src="/icons/back.gif" alt="[PARENTDIR]"></td><td><a href="/pub/Mirrors/download.mapsforge.org/maps/v4/">Parent Directory</a></td><td>&nbsp;</td><td align="right">  - </td><td>&nbsp;</td></tr>
+<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="albania.map">albania.map</a></td><td align="right">2018-08-12 06:38  </td><td align="right"> 19M</td><td>&nbsp;</td></tr>
+<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="alps.map">alps.map</a></td><td align="right">2018-07-13 00:49  </td><td align="right">1.5G</td><td>&nbsp;</td></tr>
+<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="andorra.map">andorra.map</a></td><td align="right">2018-08-12 06:40  </td><td align="right">966K</td><td>&nbsp;</td></tr>
+<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="austria.map">austria.map</a></td><td align="right">2018-07-13 04:55  </td><td align="right">358M</td><td>&nbsp;</td></tr>
+<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="azores.map">azores.map</a></td><td align="right">2018-08-12 06:43  </td><td align="right"> 10M</td><td>&nbsp;</td></tr>
+<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="belarus.map">belarus.map</a></td><td align="right">2018-07-13 05:49  </td><td align="right">161M</td><td>&nbsp;</td></tr>
+<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="belgium.map">belgium.map</a></td><td align="right">2018-01-06 05:39  </td><td align="right">224M</td><td>&nbsp;</td></tr>
+<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="bosnia-herzegovina.map">bosnia-herzegovina.map</a></td><td align="right">2018-08-12 06:54  </td><td align="right"> 56M</td><td>&nbsp;</td></tr>
+<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="bulgaria.map">bulgaria.map</a></td><td align="right">2018-08-12 07:16  </td><td align="right"> 77M</td><td>&nbsp;</td></tr>
+<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="croatia.map">croatia.map</a></td><td align="right">2018-08-12 07:33  </td><td align="right"> 90M</td><td>&nbsp;</td></tr>
+<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="cyprus.map">cyprus.map</a></td><td align="right">2018-08-12 07:39  </td><td align="right">9.0M</td><td>&nbsp;</td></tr>
+<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="czech-republic.map">czech-republic.map</a></td><td align="right">2018-07-13 10:44  </td><td align="right">443M</td><td>&nbsp;</td></tr>
+<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="denmark.map">denmark.map</a></td><td align="right">2018-08-12 08:29  </td><td align="right">177M</td><td>&nbsp;</td></tr>
+<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="estonia.map">estonia.map</a></td><td align="right">2018-08-12 08:49  </td><td align="right"> 68M</td><td>&nbsp;</td></tr>
+<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="faroe-islands.map">faroe-islands.map</a></td><td align="right">2018-08-12 08:53  </td><td align="right">2.3M</td><td>&nbsp;</td></tr>
+<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="finland.map">finland.map</a></td><td align="right">2018-07-13 22:25  </td><td align="right">309M</td><td>&nbsp;</td></tr>
+<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="france.map">france.map</a></td><td align="right">2018-07-16 10:53  </td><td align="right">2.4G</td><td>&nbsp;</td></tr>
+<tr><td valign="top"><img src="/icons/folder.gif" alt="[DIR]"></td><td><a href="france/">france/</a></td><td align="right">2018-07-26 22:20  </td><td align="right">  - </td><td>&nbsp;</td></tr>
+<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="georgia.map">georgia.map</a></td><td align="right">2018-08-12 08:59  </td><td align="right"> 34M</td><td>&nbsp;</td></tr>
+<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="germany.map">germany.map</a></td><td align="right">2018-07-18 14:10  </td><td align="right">2.0G</td><td>&nbsp;</td></tr>
+<tr><td valign="top"><img src="/icons/folder.gif" alt="[DIR]"></td><td><a href="germany/">germany/</a></td><td align="right">2018-08-29 10:54  </td><td align="right">  - </td><td>&nbsp;</td></tr>
+<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="great-britain.map">great-britain.map</a></td><td align="right">2018-08-12 18:37  </td><td align="right">711M</td><td>&nbsp;</td></tr>
+<tr><td valign="top"><img src="/icons/folder.gif" alt="[DIR]"></td><td><a href="great-britain/">great-britain/</a></td><td align="right">2018-09-07 08:03  </td><td align="right">  - </td><td>&nbsp;</td></tr>
+<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="greece.map">greece.map</a></td><td align="right">2018-08-12 19:22  </td><td align="right">131M</td><td>&nbsp;</td></tr>
+<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="hungary.map">hungary.map</a></td><td align="right">2018-08-12 19:52  </td><td align="right">117M</td><td>&nbsp;</td></tr>
+<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="iceland.map">iceland.map</a></td><td align="right">2018-08-12 20:16  </td><td align="right"> 48M</td><td>&nbsp;</td></tr>
+<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="ireland-and-northern-ireland.map">ireland-and-northern-ireland.map</a></td><td align="right">2018-08-12 20:30  </td><td align="right">102M</td><td>&nbsp;</td></tr>
+<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="isle-of-man.map">isle-of-man.map</a></td><td align="right">2018-08-20 05:39  </td><td align="right">1.9M</td><td>&nbsp;</td></tr>
+<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="italy.map">italy.map</a></td><td align="right">2018-01-12 06:13  </td><td align="right">899M</td><td>&nbsp;</td></tr>
+<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="kosovo.map">kosovo.map</a></td><td align="right">2018-08-20 05:43  </td><td align="right">8.6M</td><td>&nbsp;</td></tr>
+<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="latvia.map">latvia.map</a></td><td align="right">2018-08-20 06:03  </td><td align="right"> 51M</td><td>&nbsp;</td></tr>
+<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="liechtenstein.map">liechtenstein.map</a></td><td align="right">2018-08-20 06:06  </td><td align="right">1.5M</td><td>&nbsp;</td></tr>
+<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="lithuania.map">lithuania.map</a></td><td align="right">2018-08-20 06:29  </td><td align="right"> 89M</td><td>&nbsp;</td></tr>
+<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="luxembourg.map">luxembourg.map</a></td><td align="right">2018-08-20 06:35  </td><td align="right"> 15M</td><td>&nbsp;</td></tr>
+<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="macedonia.map">macedonia.map</a></td><td align="right">2018-08-20 06:38  </td><td align="right"> 13M</td><td>&nbsp;</td></tr>
+<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="malta.map">malta.map</a></td><td align="right">2018-08-20 06:40  </td><td align="right">2.7M</td><td>&nbsp;</td></tr>
+<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="moldova.map">moldova.map</a></td><td align="right">2018-08-20 06:45  </td><td align="right"> 25M</td><td>&nbsp;</td></tr>
+<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="monaco.map">monaco.map</a></td><td align="right">2018-08-20 06:47  </td><td align="right">204K</td><td>&nbsp;</td></tr>
+<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="montenegro.map">montenegro.map</a></td><td align="right">2018-08-20 06:50  </td><td align="right"> 15M</td><td>&nbsp;</td></tr>
+<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="netherlands.map">netherlands.map</a></td><td align="right">2018-01-23 07:30  </td><td align="right">660M</td><td>&nbsp;</td></tr>
+<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="norway.map">norway.map</a></td><td align="right">2018-01-27 16:26  </td><td align="right">955M</td><td>&nbsp;</td></tr>
+<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="poland.map">poland.map</a></td><td align="right">2018-01-13 00:23  </td><td align="right">740M</td><td>&nbsp;</td></tr>
+<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="portugal.map">portugal.map</a></td><td align="right">2018-08-20 07:07  </td><td align="right">159M</td><td>&nbsp;</td></tr>
+<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="romania.map">romania.map</a></td><td align="right">2018-08-20 09:02  </td><td align="right">165M</td><td>&nbsp;</td></tr>
+<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="serbia.map">serbia.map</a></td><td align="right">2018-08-20 09:31  </td><td align="right"> 53M</td><td>&nbsp;</td></tr>
+<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="slovakia.map">slovakia.map</a></td><td align="right">2018-08-20 10:22  </td><td align="right">134M</td><td>&nbsp;</td></tr>
+<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="slovenia.map">slovenia.map</a></td><td align="right">2018-08-20 11:00  </td><td align="right">169M</td><td>&nbsp;</td></tr>
+<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="spain.map">spain.map</a></td><td align="right">2018-01-29 22:56  </td><td align="right">458M</td><td>&nbsp;</td></tr>
+<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="sweden.map">sweden.map</a></td><td align="right">2018-01-30 02:53  </td><td align="right">365M</td><td>&nbsp;</td></tr>
+<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="switzerland.map">switzerland.map</a></td><td align="right">2018-07-20 00:28  </td><td align="right">195M</td><td>&nbsp;</td></tr>
+<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="turkey.map">turkey.map</a></td><td align="right">2018-07-20 02:20  </td><td align="right">214M</td><td>&nbsp;</td></tr>
+<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="ukraine.map">ukraine.map</a></td><td align="right">2018-02-08 04:50  </td><td align="right">403M</td><td>&nbsp;</td></tr>
+   <tr><th colspan="5"><hr></th></tr>
+</table>
+<address>Apache/2.4.34 (Fedora) Server at ftp-stud.hs-esslingen.de Port 80</address>
+</body></html>
diff --git a/getOfflineMaps.py b/getOfflineMaps.py
@@ -0,0 +1,44 @@
+# Uses Python 3.
+#
+# You will first need to run:
+#     `pip install requests PyFunctional`
+#
+# Then run:
+#     `python getOfflineMaps.py`
+#
+# and an updated offline map JSON file will be generated.
+
+# TODO: Turn this into a Gradle task... or maybe even an occasional Java execution, just like the Blog!!!!!
+
+import json
+import requests
+import re
+from functional import seq
+
+def get_mb(size):
+    unit = size[-1]
+    amount = float(size[:-1])
+    if unit == 'K':
+        return 1
+    if unit == 'G':
+        return round(amount * 1024)
+    if unit == 'M':
+        return round(amount)
+
+# At the moment, this just gets a list of current European maps.
+r = requests.get('http://ftp-stud.hs-esslingen.de/pub/Mirrors/download.mapsforge.org/maps/v4/europe/')
+
+# print(r.text)
+
+row_regex = re.compile('<tr>.*?<a href="([^"]*?\.map)">.*?(\d{4}-\d{2}-\d{2}).*?<td align="right">\s*([\d\.]*[MGK])</td>.*?</tr>')
+
+rows = row_regex.findall(r.text, re.MULTILINE)
+
+out = seq(rows) \
+    .map(lambda x: {'path': x[0], 'lastModified': x[1], 'sizeMb': get_mb(x[2])}) \
+    .list()
+
+out_json = json.dumps(out, indent=2)
+
+with open('offline_maps.json', 'w') as f:
+    f.write(out_json)
diff --git a/libraries/cyclestreets-core/build.gradle b/libraries/cyclestreets-core/build.gradle
@@ -29,6 +29,9 @@ dependencies {
   api 'com.fasterxml.jackson.core:jackson-core:2.9.6'
   api 'com.fasterxml.jackson.core:jackson-databind:2.9.6'
 
+  // Resumable downloads - for map packs
+  api "com.tonyodev.fetch2:fetch2:2.2.0-RC14"
+
   // XML / JSON conversion utilities
   implementation 'com.github.smart-fun:XmlToJson:1.4.4'
   implementation 'com.bazaarvoice.jolt:jolt-core:0.1.1'

diff --git a/libraries/cyclestreets-core/src/main/java/net/cyclestreets/offline/OfflineMapDownloadTask.kt b/libraries/cyclestreets-core/src/main/java/net/cyclestreets/offline/OfflineMapDownloadTask.kt
@@ -0,0 +1,60 @@
+package net.cyclestreets.offline
+
+import android.os.AsyncTask
+import android.widget.ProgressBar
+import android.widget.TextView
+import okio.Okio
+import okhttp3.OkHttpClient
+import okhttp3.Request
+import java.io.File
+import java.lang.ref.WeakReference
+
+class OfflineMapDownloadTask(private val url: String,
+                             private val destFile: File,
+                             val progressBar: WeakReference<ProgressBar>,
+                             val localInfo: WeakReference<TextView>,
+                             val localDelete: WeakReference<TextView>) : AsyncTask<Void, Int, Boolean>() {
+
+    override fun doInBackground(vararg params: Void?): Boolean {
+        val request = Request.Builder().url(url).build()
+        val response = client.newCall(request).execute()
+        val contentLength = response.body()!!.contentLength()
+        val source = response.body()!!.source()
+
+        val sink = Okio.buffer(Okio.sink(destFile))
+        val sinkBuffer = sink.buffer()
+
+        var totalBytesRead = 0L
+        val bufferSize = 8 * 1024L
+        var bytesRead: Long = source.read(sinkBuffer, bufferSize)
+
+        while (bytesRead != -1L && !isCancelled) {
+            sink.emit()
+            totalBytesRead += bytesRead
+            publishProgress((totalBytesRead * 100 / contentLength).toInt())
+            bytesRead = source.read(sinkBuffer, bufferSize)
+        }
+
+        sink.flush()
+        sink.close()
+        source.close()
+        return true
+    }
+
+    override fun onProgressUpdate(vararg values: Int?) {
+        val percentDone = values[0]
+    }
+
+    override fun onPostExecute(result: Boolean?) {
+        // TODO: mark successful in all ways
+    }
+
+    override fun onCancelled(result: Boolean?) {
+        if (result == false)
+            destFile.delete()
+    }
+
+    companion object {
+        val client: OkHttpClient = OkHttpClient.Builder().build()
+    }
+}
diff --git a/libraries/cyclestreets-core/src/main/java/net/cyclestreets/offline/OfflineMapInfoTask.kt b/libraries/cyclestreets-core/src/main/java/net/cyclestreets/offline/OfflineMapInfoTask.kt
@@ -0,0 +1,67 @@
+package net.cyclestreets.offline
+
+import android.content.Context
+import android.util.Log
+import net.cyclestreets.util.Logging
+import okhttp3.OkHttpClient
+import okhttp3.Request
+import java.io.IOException
+import java.util.*
+
+//TODO: save stuff into a DB table, or a serialized String preference?
+
+private val TAG = Logging.getTag(OfflineMapInfoTask::class.java)
+
+private val apacheUrls = listOf("http://ftp-stud.hs-esslingen.de/pub/Mirrors/download.mapsforge.org/maps/v4/europe/")
+private val regex = Regex("""<tr>.*?<a href="([^"]*?\.map)">.*?(\d{4}-\d{2}-\d{2}).*?<td align="right">\s*([\d\.]*[MGK])</td>.*?</tr>""")
+
+internal fun parse(apacheIndexPage: String): Sequence<MatchResult.Destructured> {
+    return regex.findAll(apacheIndexPage).map { r -> r.destructured }
+}
+
+internal fun offlineMapFor(apacheUrl: String, dmr: MatchResult.Destructured): OfflineMap {
+    return OfflineMap(supportedMaps[dmr.component1()] ?: dmr.component1().dropLast(4).capitalize(),
+                      "$apacheUrl${dmr.component1()}",
+                      dmr.component2(),
+                      getMb(dmr.component3()))
+}
+
+private fun getMb(size: String): Int {
+    val amount = size.dropLast(1).toFloat()
+    return when (size.last()) {
+        'K' -> 1
+        'G' -> Math.round(amount * 1024)
+        'M' -> Math.round (amount)
+        else -> -1
+    }
+}
+
+class OfflineMapInfoTask(private val context: Context) : TimerTask() {
+    override fun run() {
+        val fred: List<OfflineMap> = apacheUrls.flatMap { url -> offlineMapsAt(url) }
+
+    }
+
+    private fun offlineMapsAt(apacheUrl: String): List<OfflineMap> {
+        val apacheIndexPage = restGet(apacheUrl)
+        return parse(apacheIndexPage)
+                .filter { e -> e.component1() in supportedMaps }
+                .map { e ->  offlineMapFor(apacheUrl, e)}
+                .toList()
+    }
+
+    private fun restGet(url: String): String {
+        val request = Request.Builder().url(url).build()
+        return try {
+            val response = client.newCall(request).execute()
+            response.body()?.string() ?: ""
+        } catch (e: IOException) {
+            Log.w(TAG, "Failed to list offline maps at $url")
+            ""
+        }
+    }
+
+    companion object {
+        val client: OkHttpClient = OkHttpClient.Builder().build()
+    }
+}
diff --git a/libraries/cyclestreets-core/src/main/java/net/cyclestreets/offline/OfflineMaps.kt b/libraries/cyclestreets-core/src/main/java/net/cyclestreets/offline/OfflineMaps.kt
@@ -0,0 +1,23 @@
+package net.cyclestreets.offline
+
+private const val downloadRoot = "http://ftp-stud.hs-esslingen.de/pub/Mirrors/download.mapsforge.org/maps/v4/europe/"
+
+//val offlineMaps: List<OfflineMap> = listOf(
+//    OfflineMap("England",     505, "europe/great-britain/england"),
+//    OfflineMap("Scotland",    110, "europe/great-britain/scotland"),
+//    OfflineMap("Wales",        47, "europe/great-britain/wales"),
+//    OfflineMap("Isle of Man",   2, "europe/isle-of-man")
+//)
+
+internal val supportedMaps = mapOf(
+    "isle-of-man.map" to "Isle of Man",
+    "monaco.map" to "Monaco"
+)
+
+class OfflineMap(val name: String, val url: String, val lastModified: String, val sizeMb: Int) {
+    override fun toString(): String {
+        return "OfflineMap(name='$name', url='$url', lastModified='$lastModified', sizeMb=$sizeMb)"
+    }
+
+
+}
diff --git a/...estreets-core/src/test/java/net/cyclestreets/api/client/JourneyStringTransformerTest.java b/...estreets-core/src/test/java/net/cyclestreets/api/client/JourneyStringTransformerTest.java
@@ -14,6 +14,7 @@
 
 import java.io.IOException;
 
+@Ignore
 @Config(manifest= Config.NONE, sdk = 23)
 @RunWith(RobolectricTestRunner.class)
 public class JourneyStringTransformerTest {

diff --git a/...es/cyclestreets-core/src/test/java/net/cyclestreets/api/client/RetrofitApiClientTest.java b/...es/cyclestreets-core/src/test/java/net/cyclestreets/api/client/RetrofitApiClientTest.java
@@ -25,6 +25,7 @@
 
 import org.apache.commons.io.FileUtils;
 import org.junit.Before;
+import org.junit.Ignore;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.runner.RunWith;
@@ -57,6 +58,7 @@
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
 
+@Ignore
 @Config(manifest=Config.NONE, sdk = 23)
 @RunWith(RobolectricTestRunner.class)
 public class RetrofitApiClientTest {