fix: OSD: concatenate multiline typical pedon horizons

R/parseOSD_functions.R

@@ -357,10 +357,21 @@
     hz.idx <- hz.idx[-first.line.flag]
   }
 
-  check.multiline <- diff(hz.idx) > 1
-  if (any(check.multiline)) {
+  check.multiline <- diff(hz.idx)
+  if (any(check.multiline > 1)) {
     # multiline typical pedon horizon formatting (needs fix)
-    logmsg(logfile, paste0("CHECK MULTILINE TYPICAL PEDON: ", filename, " [number of multilines=", sum(check.multiline), "]"))
+    logmsg(logfile, paste0("CHECK MULTILINE TYPICAL PEDON: ", filename, " [number of multilines=", sum(check.multiline > 1), "]"))
+  }
+
+  if (length(hz.idx) > 0 ) {
+    # concatenate multiline entries in typical pedon
+    multiline.rle <- list(lengths = c(check.multiline, 1),
+                          values = hz.idx)
+    class(multiline.rle) <- "rle"
+    grp <- inverse.rle(multiline.rle)
+    tpclean <- sapply(split(tp[min(grp):max(grp)], grp), paste, collapse = " ")
+  } else {
+    tpclean <- tp[0]
   }
 
   # init empty lists to store hz data and colors
@@ -370,8 +381,8 @@
   narrative.data <- list()
 
   # iterate over identified horizons, extracting hz parts
-  for (i in seq_along(hz.idx)) {
-    this.chunk <- tp[hz.idx[i]]
+  for (i in seq_along(tpclean)) {
+    this.chunk <- tpclean[i]
 
     # parse hz designations and depths, keep first match
     # first try to find horizons with top AND bottom depths
@@ -408,26 +419,26 @@
     if (!is.na(h[1]) && grepl("[OABCDELMRVWbcxw]", h[1])) {
       # save hz data to list
       hz.data[[i]] <- h
-  
+
       # save narrative to list
       narrative.data[[i]] <- this.chunk
-      
+
       ## TODO: test this!
       # parse ALL colors, result is a multi-row matrix, 5th column is moisture state
       colors <- stringi::stri_match_all(this.chunk, regex = color.rule)[[1]]
-  
+
       # replace missing moisture state with (parsed) default value
       colors[, 5][which(colors[, 5] == '')] <- default.moisture.state
-  
+
       # extract dry|moist colors, note that there may be >1 color per state
       dc <- colors[which(colors[, 5] == 'dry'), 1:4, drop = FALSE]
       mc <- colors[which(colors[, 5] == 'moist'), 1:4, drop = FALSE]
-  
+
       # there there was at least 1 match, keep the first 1
       if (nrow(dc) > 0) {
         dry.colors[[i]] <- dc[1, ]
       } else dry.colors[[i]] <- matrix(rep(NA, times = 4), nrow = 1)
-  
+
       if (nrow(mc) > 0)
         moist.colors[[i]] <- mc[1, ]
       else moist.colors[[i]] <- matrix(rep(NA, times = 4), nrow = 1)
@@ -437,7 +448,7 @@
       dry.colors[[i]] <- NULL
       moist.colors[[i]] <- NULL
     }
-    
+
   }
 
   # test for no parsed data, must be some funky formatting...

inst/extdata/OSD-error-reporting/RO/bozeman-hz.csv

@@ -49,7 +49,7 @@
 "BENSLEY","Bw3",79,114,"l0YR","6","2","l0YR","4","2"
 "BENSLEY","BC",114,160,"l0YR","6","3","l0YR","4","3"
 "BENTILLA","Oi",5,0,NA,NA,NA,NA,NA,NA
-"BICKETT","2Bkgl",28,61,NA,NA,NA,"2.5Y","6","1"
+"BICKETT","2Bkgl",28,61,"2.5Y","7","1","2.5Y","6","1"
 "BIRDSVIEW","A",0,13,"l0YR","5","4","l0YR","3","3"
 "BIRDSVIEW","O",10,0,NA,NA,NA,NA,NA,NA
 "BIRDSVIEW","Bwl",13,76,NA,NA,NA,"l0YR","4","4"
@@ -178,7 +178,7 @@
 "DOMELL","Bw1",28,58,"l0YR","6","4","l0YR","3","3"
 "DOMELL","Bw2",58,94,"l0YR","7","4","l0YR","4","4"
 "EASTPINE","Oi",3,0,NA,NA,NA,NA,NA,NA
-"EDFRO","Bsl",18,28,NA,NA,NA,"7.5YR","4","6"
+"EDFRO","Bsl",18,28,"7.5YR","5","8","7.5YR","4","6"
 "EDGEWICK","Bwl",20,25,"2.5Y","7","2","2.5Y","4","4"
 "EDSON","Oi",5,0,NA,NA,NA,NA,NA,NA
 "EGYPTCREEK","Oi",3,0,NA,NA,NA,NA,NA,NA
@@ -253,7 +253,7 @@
 "HOOTER","Oe",5,0,NA,NA,NA,NA,NA,NA
 "HOOTER","Oi",8,5,NA,NA,NA,NA,NA,NA
 "HOPBURN","Oi",1,0,NA,NA,NA,NA,NA,NA
-"HOVDE","A",0,23,NA,NA,NA,"l0YR","3","2"
+"HOVDE","A",0,23,"2.5Y","6","2","l0YR","3","2"
 "HOVDE","Oi",3,0,NA,NA,NA,NA,NA,NA
 "HOVDE","2C2",33,74,"l0YR","5","3","l0YR","4","3"
 "HOVDE","2C4",99,152,"l0YR","4","2","l0YR","3","2"
@@ -667,11 +667,11 @@
 "WARRENTON","O",8,0,NA,NA,NA,"N","2",NA
 "WASHOUGAL","O1",3,0,NA,NA,NA,NA,NA,NA
 "WEDGEMONT","Bsl",5,51,"10YR","7","3","10YR","5","3"
-"WELCOME","E",0,5,NA,NA,NA,"l0YR","4","3"
+"WELCOME","E",0,5,"l0YR","7","2","l0YR","4","3"
 "WELCOME","Oa",5,0,NA,NA,NA,NA,NA,NA
-"WELCOME","Bs1",5,15,NA,NA,NA,"l0YR","3","4"
+"WELCOME","Bs1",5,15,"l0YR","6","4","l0YR","3","4"
 "WELCOME","Oi",8,5,NA,NA,NA,NA,NA,NA
-"WELCOME","Bs2",15,48,NA,NA,NA,"l0YR","3","6"
+"WELCOME","Bs2",15,48,"l0YR","7","4","l0YR","3","6"
 "WELLSBENCH","Btl",15,36,"10YR","4","2","10YR","3","3"
 "WESSEL","O1 and O2",5,0,NA,NA,NA,NA,NA,NA
 "WHETSTONE","O2",5,0,NA,NA,NA,NA,NA,NA

inst/extdata/OSD-error-reporting/log.txt

@@ -257,7 +257,7 @@
   UT   219   11
   VA    30    0
   VT     8    0
-  WA   249   39
+  WA   248   40
   WI     8    0
   WV     4    0
   WY   212    1

inst/extdata/OSD-error-reporting/state/ID-hz.csv

@@ -1,5 +1,5 @@
 "id","name","top","bottom","dry_hue","dry_value","dry_chroma","moist_hue","moist_value","moist_chroma"
-"BICKETT","2Bkgl",28,61,NA,NA,NA,"2.5Y","6","1"
+"BICKETT","2Bkgl",28,61,"2.5Y","7","1","2.5Y","6","1"
 "BUTTECREEK","Oe",5,0,NA,NA,NA,NA,NA,NA
 "CHICANE","A12",13,36,"lOYR","3","2","lOYR","2","1"
 "CHICANE","B2",36,53,"lOYR","4","2","lOYR","3","1"

inst/extdata/OSD-error-reporting/state/WA-hz.csv

@@ -114,7 +114,7 @@
 "DOMELL","A2",13,28,"l0YR","5","3","l0YR","3","3"
 "DOMELL","Bw1",28,58,"l0YR","6","4","l0YR","3","3"
 "DOMELL","Bw2",58,94,"l0YR","7","4","l0YR","4","4"
-"EDFRO","Bsl",18,28,NA,NA,NA,"7.5YR","4","6"
+"EDFRO","Bsl",18,28,"7.5YR","5","8","7.5YR","4","6"
 "EDGEWICK","Bwl",20,25,"2.5Y","7","2","2.5Y","4","4"
 "ELWELL","O2",3,0,NA,NA,NA,NA,NA,NA
 "ELWELL","O1",5,3,NA,NA,NA,NA,NA,NA
@@ -150,7 +150,7 @@
 "HOFFSTADT","Oe",5,0,NA,NA,NA,NA,NA,NA
 "HOODSPORT","Oe",3,0,NA,NA,NA,"5YR","2","1"
 "HOODSPORT","Oi",6,3,NA,NA,NA,NA,NA,NA
-"HOVDE","A",0,23,NA,NA,NA,"l0YR","3","2"
+"HOVDE","A",0,23,"2.5Y","6","2","l0YR","3","2"
 "HOVDE","Oi",3,0,NA,NA,NA,NA,NA,NA
 "HOVDE","2C2",33,74,"l0YR","5","3","l0YR","4","3"
 "HOVDE","2C4",99,152,"l0YR","4","2","l0YR","3","2"
@@ -424,11 +424,11 @@
 "WALVILLE","Al2",30,43,"7.5YR","5","4","7.5YR","3","4"
 "WASHOUGAL","O1",3,0,NA,NA,NA,NA,NA,NA
 "WEDGEMONT","Bsl",5,51,"10YR","7","3","10YR","5","3"
-"WELCOME","E",0,5,NA,NA,NA,"l0YR","4","3"
+"WELCOME","E",0,5,"l0YR","7","2","l0YR","4","3"
 "WELCOME","Oa",5,0,NA,NA,NA,NA,NA,NA
-"WELCOME","Bs1",5,15,NA,NA,NA,"l0YR","3","4"
+"WELCOME","Bs1",5,15,"l0YR","6","4","l0YR","3","4"
 "WELCOME","Oi",8,5,NA,NA,NA,NA,NA,NA
-"WELCOME","Bs2",15,48,NA,NA,NA,"l0YR","3","6"
+"WELCOME","Bs2",15,48,"l0YR","7","4","l0YR","3","6"
 "WILKESON","Al",0,10,"l0YR","5","2","l0YR","3","2"
 "WILKESON","A2",10,25,"l0YR","5","3","l0YR","3","3"
 "WILKESON","BA",25,46,"l0YR","5","3","l0YR","4","3"

inst/extdata/OSD/A/ACKWATER.json

@@ -134,11 +134,11 @@
         "coherence": "NA",
         "cf_class": "NA",
         "pH": "NA",
-        "pH_class": "NA",
+        "pH_class": "very strongly acid",
         "eff_class": "NA",
-        "distinctness": "NA",
-        "topography": "NA",
-        "narrative": "Bt2--9 to 16 inches; yellowish brown (lOYR 5/8) silty clay loam; moderate medium subangular blocky structure; firm, sticky, plastic; few fine roots; few very fine pores; common prominent clay films on"
+        "distinctness": "gradual",
+        "topography": "smooth",
+        "narrative": "Bt2--9 to 16 inches; yellowish brown (lOYR 5/8) silty clay loam; moderate medium subangular blocky structure; firm, sticky, plastic; few fine roots; few very fine pores; common prominent clay films on faces of peds; very strongly acid; gradual smooth boundary."
       },
       {
         "name": "Bt3",
@@ -157,11 +157,11 @@
         "coherence": "NA",
         "cf_class": "NA",
         "pH": "NA",
-        "pH_class": "NA",
+        "pH_class": "very strongly acid",
         "eff_class": "NA",
-        "distinctness": "NA",
-        "topography": "NA",
-        "narrative": "Bt3--16 to 28 inches; strong brown (7.5YR 5/8) silty clay; many coarse distinct light gray (lOYR 7/1) iron depletions and common medium prominent red 2.5YR 4/8) masses of iron accumulation; weak coarse prismatic structure parting to moderate medium subangular blocky; firm, sticky, plastic; few fine roots; few very fine pores; common prominent clay films on faces of peds;"
+        "distinctness": "gradual",
+        "topography": "wavy",
+        "narrative": "Bt3--16 to 28 inches; strong brown (7.5YR 5/8) silty clay; many coarse distinct light gray (lOYR 7/1) iron depletions and common medium prominent red 2.5YR 4/8) masses of iron accumulation; weak coarse prismatic structure parting to moderate medium subangular blocky; firm, sticky, plastic; few fine roots; few very fine pores; common prominent clay films on faces of peds; very strongly acid; gradual wavy boundary."
       },
       {
         "name": "Bt4",
@@ -180,11 +180,11 @@
         "coherence": "NA",
         "cf_class": "NA",
         "pH": "NA",
-        "pH_class": "NA",
+        "pH_class": "very strongly acid",
         "eff_class": "NA",
-        "distinctness": "NA",
-        "topography": "NA",
-        "narrative": "Bt4--28 to 52 inches; mottled strong brown (7.5YR 5/8), light gray (lOYR 7/1), red (2.5YR 4/8), and brownish yellow (lOYR 6/8) silty clay; moderate coarse prismatic structure parting to moderate very thick platy parting to moderate medium angular blocky; firm, sticky, plastic; few very fine roots along faces of peds; common prominent clay"
+        "distinctness": "gradual",
+        "topography": "smooth",
+        "narrative": "Bt4--28 to 52 inches; mottled strong brown (7.5YR 5/8), light gray (lOYR 7/1), red (2.5YR 4/8), and brownish yellow (lOYR 6/8) silty clay; moderate coarse prismatic structure parting to moderate very thick platy parting to moderate medium angular blocky; firm, sticky, plastic; few very fine roots along faces of peds; common prominent clay films on faces of peds; very strongly acid; gradual smooth boundary."
       },
       {
         "name": "Btg",

inst/extdata/OSD/A/ACUNA.json

@@ -82,17 +82,17 @@
         "moist_value": 3,
         "moist_chroma": 2,
         "texture_class": "silty clay",
-        "structure": "NA",
-        "dry_rupture": "NA",
-        "moist_rupture": "NA",
+        "structure": "moderate medium subangular blocky",
+        "dry_rupture": "hard",
+        "moist_rupture": "friable",
         "coherence": "NA",
         "cf_class": "NA",
         "pH": "NA",
-        "pH_class": "NA",
+        "pH_class": "moderately alkaline",
         "eff_class": "NA",
-        "distinctness": "NA",
-        "topography": "NA",
-        "narrative": "A1--0 to 18 inches; dark grayish brown (10YR 4/2) silty clay, very dark grayish brown (10YR 3/2) moist; the upper 2 inches is grayish brown (10YR 5/2); moderate medium subangular blocky"
+        "distinctness": "clear",
+        "topography": "wavy",
+        "narrative": "A1--0 to 18 inches; dark grayish brown (10YR 4/2) silty clay, very dark grayish brown (10YR 3/2) moist; the upper 2 inches is grayish brown (10YR 5/2); moderate medium subangular blocky structure parting to moderate medium granular; slightly hard, friable; common fine and medium roots; common fine pores; few fine threads and films of calcium carbonate; few fine calcium carbonate concretions less than 2 mm across; calcareous; moderately alkaline; clear wavy boundary.  (10 to 20 inches thick)"
       },
       {
         "name": "B21",
@@ -101,21 +101,21 @@
         "dry_hue": "7.5YR",
         "dry_value": 5,
         "dry_chroma": 4,
-        "moist_hue": "NA",
-        "moist_value": "NA",
-        "moist_chroma": "NA",
+        "moist_hue": "7.5YR",
+        "moist_value": 4,
+        "moist_chroma": 4,
         "texture_class": "clay",
-        "structure": "NA",
-        "dry_rupture": "NA",
-        "moist_rupture": "NA",
+        "structure": "moderate medium subangular blocky",
+        "dry_rupture": "hard",
+        "moist_rupture": "friable",
         "coherence": "NA",
         "cf_class": "NA",
         "pH": "NA",
-        "pH_class": "NA",
+        "pH_class": "moderately alkaline",
         "eff_class": "NA",
-        "distinctness": "NA",
-        "topography": "NA",
-        "narrative": "B21--18 to 30 inches; brown (7.5YR 5/4) clay, dark brown"
+        "distinctness": "abrupt",
+        "topography": "wavy",
+        "narrative": "B21--18 to 30 inches; brown (7.5YR 5/4) clay, dark brown (7.5YR 4/4) moist; moderate medium subangular blocky structure parting to moderate medium granular; hard, friable; few fine roots; common very fine pores; few films and threads of calcium carbonate; few thin discontinuous clay films on vertical ped surfaces; calcareous; moderately alkaline; abrupt wavy boundary. (6 to 28 inches thick)"
       },
       {
         "name": "B22ca",
@@ -124,21 +124,21 @@
         "dry_hue": "7.5R",
         "dry_value": 7,
         "dry_chroma": 4,
-        "moist_hue": "NA",
-        "moist_value": "NA",
-        "moist_chroma": "NA",
+        "moist_hue": "7.5YR",
+        "moist_value": 5,
+        "moist_chroma": 4,
         "texture_class": "silty clay",
-        "structure": "NA",
-        "dry_rupture": "NA",
-        "moist_rupture": "NA",
+        "structure": "weak fine subangular blocky",
+        "dry_rupture": "hard",
+        "moist_rupture": "friable",
         "coherence": "NA",
         "cf_class": "NA",
         "pH": "NA",
-        "pH_class": "NA",
+        "pH_class": "moderately alkaline",
         "eff_class": "NA",
-        "distinctness": "NA",
-        "topography": "NA",
-        "narrative": "B22ca--30 to 38 inches; pink (7.5R 7/4) silty clay, brown"
+        "distinctness": "clear",
+        "topography": "wavy",
+        "narrative": "B22ca--30 to 38 inches; pink (7.5R 7/4) silty clay, brown (7.5YR 5/4) moist; weak fine subangular blocky structure; slightly hard; friable; many very fine pores; 15 percent by volume of threads, films, and concretions 2 mm to 2 cm across of calcium carbonate; calcareous; moderately alkaline; clear wavy boundary. (8 to 28 inches thick)"
       },
       {
         "name": "B23ca",
@@ -152,16 +152,16 @@
         "moist_chroma": 6,
         "texture_class": "silty clay",
         "structure": "weak fine subangular blocky",
-        "dry_rupture": "hard",
+        "dry_rupture": "soft",
         "moist_rupture": "friable",
         "coherence": "NA",
         "cf_class": "NA",
         "pH": "NA",
-        "pH_class": "NA",
+        "pH_class": "moderately alkaline",
         "eff_class": "NA",
-        "distinctness": "NA",
-        "topography": "NA",
-        "narrative": "B23ca--38 to 60 inches; pink (7.5YR 8/4) silty clay, reddish yellow (7.5YR 7/6) moist; weak fine subangular blocky structure; slightly hard, friable; common very fine pores; 8 percent by"
+        "distinctness": "gradual",
+        "topography": "wavy",
+        "narrative": "B23ca--38 to 60 inches; pink (7.5YR 8/4) silty clay, reddish yellow (7.5YR 7/6) moist; weak fine subangular blocky structure; slightly hard, friable; common very fine pores; 8 percent by volume of threads, films and soft masses of 2 mm to 2 cm across of calcium carbonate, calcareous; moderately alkaline; gradual wavy boundary.  (10 to 25 inches thick)"
       },
       {
         "name": "B24ca",