Skip to content

Commit 0d8f24f

Browse files
SamCarlbergSamCarlberg
authored
Add cascade classifier operation (#678)
* Add cascade classfier operation Also adds a new socket input view for file selection * Add FileSource, remove file sockets * Move Mat drawing to a utility class * Fix MatUtil rename * Add code generation * Revert MatUtil commits This reverts commit 08e53d3. Revert "Fix MatUtil rename" This reverts commit 94c486b. * Change generic "FileSource" to specialized "ClassifierSource" * Add scale factor, min neighbors, and min/max size to cascade classifier * Update code generation * Use pre-existing hints for size sockets * Fix a bug when trying to fall back to the previous file * Make classifier source supply a classifier object Add missing import in generated Java code * Throw IOException if classifier file is invalid
1 parent 31e6b93 commit 0d8f24f

File tree

22 files changed

+560
-13
lines changed

22 files changed

+560
-13
lines changed

‎build.gradle‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -295,8 +295,6 @@ project(":core") {
295295
use any more of these (or perhaps just include them for people to use from Python scripts), the following lines
296296
should be changed, but for now this saves us a lot of space. */
297297
exclude 'org/bytedeco/javacpp/*/*calib3d*'
298-
exclude 'org/bytedeco/javacpp/*/*face*'
299-
exclude 'org/bytedeco/javacpp/*/*objdetect*'
300298
exclude 'org/bytedeco/javacpp/*/*optflow*'
301299
exclude 'org/bytedeco/javacpp/*/*photo*'
302300
exclude 'org/bytedeco/javacpp/*/*shape*'

‎core/src/main/java/edu/wpi/grip/core/GripCoreModule.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010
import edu.wpi.grip.core.sockets.OutputSocket;
1111
import edu.wpi.grip.core.sockets.OutputSocketImpl;
1212
import edu.wpi.grip.core.sources.CameraSource;
13+
import edu.wpi.grip.core.sources.ClassifierSource;
1314
import edu.wpi.grip.core.sources.HttpSource;
1415
import edu.wpi.grip.core.sources.ImageFileSource;
1516
import edu.wpi.grip.core.sources.MultiImageFileSource;
@@ -151,6 +152,9 @@ public <I> void hear(TypeLiteral<I> type, TypeEncounter<I> encounter) {
151152
install(new FactoryModuleBuilder()
152153
.implement(NetworkTableEntrySource.class, NetworkTableEntrySource.class)
153154
.build(NetworkTableEntrySource.Factory.class));
155+
install(new FactoryModuleBuilder()
156+
.implement(ClassifierSource.class, ClassifierSource.class)
157+
.build(ClassifierSource.Factory.class));
154158

155159
install(new FactoryModuleBuilder().build(ExceptionWitness.Factory.class));
156160
install(new FactoryModuleBuilder().build(Timer.Factory.class));

‎core/src/main/java/edu/wpi/grip/core/Source.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22

33
import edu.wpi.grip.core.sockets.OutputSocket;
44
import edu.wpi.grip.core.sources.CameraSource;
5+
import edu.wpi.grip.core.sources.ClassifierSource;
56
import edu.wpi.grip.core.sources.HttpSource;
67
import edu.wpi.grip.core.sources.ImageFileSource;
78
import edu.wpi.grip.core.sources.MultiImageFileSource;
@@ -114,6 +115,8 @@ public static class SourceFactoryImpl implements SourceFactory {
114115
HttpSource.Factory httpFactory;
115116
@Inject
116117
NetworkTableEntrySource.Factory networkTableEntryFactory;
118+
@Inject
119+
ClassifierSource.Factory fileSourceFactory;
117120

118121
@Override
119122
public Source create(Class<?> type, Properties properties) throws IOException {
@@ -127,6 +130,8 @@ public Source create(Class<?> type, Properties properties) throws IOException {
127130
return httpFactory.create(properties);
128131
} else if (type.isAssignableFrom(NetworkTableEntrySource.class)) {
129132
return networkTableEntryFactory.create(properties);
133+
} else if (type.isAssignableFrom(ClassifierSource.class)) {
134+
return fileSourceFactory.create(properties);
130135
} else {
131136
throw new IllegalArgumentException(type + " was not a valid type");
132137
}

‎core/src/main/java/edu/wpi/grip/core/operations/Operations.java‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
import edu.wpi.grip.core.events.OperationAddedEvent;
66
import edu.wpi.grip.core.operations.composite.BlobsReport;
77
import edu.wpi.grip.core.operations.composite.BlurOperation;
8+
import edu.wpi.grip.core.operations.composite.CascadeClassifierOperation;
89
import edu.wpi.grip.core.operations.composite.ContoursReport;
910
import edu.wpi.grip.core.operations.composite.ConvexHullsOperation;
1011
import edu.wpi.grip.core.operations.composite.DesaturateOperation;
@@ -21,6 +22,7 @@
2122
import edu.wpi.grip.core.operations.composite.NormalizeOperation;
2223
import edu.wpi.grip.core.operations.composite.PublishVideoOperation;
2324
import edu.wpi.grip.core.operations.composite.RGBThresholdOperation;
25+
import edu.wpi.grip.core.operations.composite.RectsReport;
2426
import edu.wpi.grip.core.operations.composite.ResizeOperation;
2527
import edu.wpi.grip.core.operations.composite.SaveImageOperation;
2628
import edu.wpi.grip.core.operations.composite.SwitchOperation;
@@ -119,6 +121,8 @@ public class Operations {
119121
() -> new WatershedOperation(isf, osf)),
120122
new OperationMetaData(ThresholdMoving.DESCRIPTION,
121123
() -> new ThresholdMoving(isf, osf)),
124+
new OperationMetaData(CascadeClassifierOperation.DESCRIPTION,
125+
() -> new CascadeClassifierOperation(isf, osf)),
122126

123127
// OpenCV operations
124128
new OperationMetaData(MatFieldAccessor.DESCRIPTION,
@@ -149,6 +153,8 @@ public class Operations {
149153
new OperationMetaData(NTPublishAnnotatedOperation.descriptionFor(Boolean.class),
150154
() -> new NTPublishAnnotatedOperation<>(isf, Boolean.class, BooleanPublishable.class,
151155
BooleanPublishable::new, ntPublisherFactory)),
156+
new OperationMetaData(NTPublishAnnotatedOperation.descriptionFor(RectsReport.class),
157+
() -> new NTPublishAnnotatedOperation<>(isf, RectsReport.class, ntPublisherFactory)),
152158

153159
// ROS publishing operations
154160
new OperationMetaData(ROSPublishOperation.descriptionFor(Number.class),
@@ -185,7 +191,9 @@ public class Operations {
185191
NumberPublishable::new, httpPublishFactory)),
186192
new OperationMetaData(HttpPublishOperation.descriptionFor(Boolean.class),
187193
() -> new HttpPublishOperation<>(isf, Boolean.class, BooleanPublishable.class,
188-
BooleanPublishable::new, httpPublishFactory))
194+
BooleanPublishable::new, httpPublishFactory)),
195+
new OperationMetaData(HttpPublishOperation.descriptionFor(RectsReport.class),
196+
() -> new HttpPublishOperation<>(isf, RectsReport.class, httpPublishFactory))
189197
);
190198
}
191199

‎core/src/main/java/edu/wpi/grip/core/operations/composite/CascadeClassifierOperation.java‎

Lines changed: 116 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,116 @@
1+
package edu.wpi.grip.core.operations.composite;
2+
3+
import edu.wpi.grip.core.Operation;
4+
import edu.wpi.grip.core.OperationDescription;
5+
import edu.wpi.grip.core.sockets.InputSocket;
6+
import edu.wpi.grip.core.sockets.OutputSocket;
7+
import edu.wpi.grip.core.sockets.SocketHint;
8+
import edu.wpi.grip.core.sockets.SocketHints;
9+
import edu.wpi.grip.core.util.Icon;
10+
11+
import com.google.common.collect.ImmutableList;
12+
13+
import org.bytedeco.javacpp.opencv_core.Mat;
14+
import org.bytedeco.javacpp.opencv_core.Rect;
15+
import org.bytedeco.javacpp.opencv_core.RectVector;
16+
import org.bytedeco.javacpp.opencv_core.Size;
17+
import org.bytedeco.javacpp.opencv_objdetect.CascadeClassifier;
18+
19+
import java.util.ArrayList;
20+
import java.util.List;
21+
22+
/**
23+
* Operation for identifying parts of an image with a cascade classifier.
24+
*/
25+
public class CascadeClassifierOperation implements Operation {
26+
27+
public static final OperationDescription DESCRIPTION =
28+
OperationDescription.builder()
29+
.name("Cascade Classifier")
30+
.summary("Runs a cascade classifier on an image")
31+
.icon(Icon.iconStream("opencv"))
32+
.category(OperationDescription.Category.FEATURE_DETECTION)
33+
.build();
34+
35+
private final SocketHint<Mat> imageHint =
36+
SocketHints.Inputs.createMatSocketHint("Image", false);
37+
private final SocketHint<CascadeClassifier> classifierHint =
38+
new SocketHint.Builder<>(CascadeClassifier.class)
39+
.identifier("Classifier")
40+
.build();
41+
private final SocketHint<Number> scaleHint =
42+
SocketHints.Inputs.createNumberSpinnerSocketHint("Scale factor", 1.1, 1.01, Double.MAX_VALUE);
43+
private final SocketHint<Number> minNeighborsHint =
44+
SocketHints.Inputs.createNumberSpinnerSocketHint("Min neighbors", 3, 0, Integer.MAX_VALUE);
45+
private final SocketHint<Size> minSizeHint =
46+
SocketHints.Inputs.createSizeSocketHint("Min size", true);
47+
private final SocketHint<Size> maxSizeHint =
48+
SocketHints.Inputs.createSizeSocketHint("Max size", true);
49+
private final SocketHint<RectsReport> outputHint =
50+
new SocketHint.Builder<>(RectsReport.class)
51+
.identifier("Detected areas")
52+
.initialValue(RectsReport.NIL)
53+
.build();
54+
55+
private final InputSocket<Mat> imageSocket;
56+
private final InputSocket<CascadeClassifier> classifierSocket;
57+
private final InputSocket<Number> scaleSocket;
58+
private final InputSocket<Number> minNeighborsSocket;
59+
private final InputSocket<Size> minSizeSocket;
60+
private final InputSocket<Size> maxSizeSocket;
61+
private final OutputSocket<RectsReport> output;
62+
63+
@SuppressWarnings("JavadocMethod")
64+
public CascadeClassifierOperation(InputSocket.Factory isf, OutputSocket.Factory osf) {
65+
imageSocket = isf.create(imageHint);
66+
classifierSocket = isf.create(classifierHint);
67+
scaleSocket = isf.create(scaleHint);
68+
minNeighborsSocket = isf.create(minNeighborsHint);
69+
minSizeSocket = isf.create(minSizeHint);
70+
maxSizeSocket = isf.create(maxSizeHint);
71+
output = osf.create(outputHint);
72+
}
73+
74+
@Override
75+
public List<InputSocket> getInputSockets() {
76+
return ImmutableList.of(
77+
imageSocket,
78+
classifierSocket,
79+
scaleSocket,
80+
minNeighborsSocket,
81+
minSizeSocket,
82+
maxSizeSocket
83+
);
84+
}
85+
86+
@Override
87+
public List<OutputSocket> getOutputSockets() {
88+
return ImmutableList.of(
89+
output
90+
);
91+
}
92+
93+
@Override
94+
public void perform() {
95+
if (!imageSocket.getValue().isPresent() || !classifierSocket.getValue().isPresent()) {
96+
return;
97+
}
98+
final Mat image = imageSocket.getValue().get();
99+
if (image.empty() || image.channels() != 3) {
100+
throw new IllegalArgumentException("A cascade classifier needs a three-channel input");
101+
}
102+
final CascadeClassifier classifier = classifierSocket.getValue().get();
103+
final double scaleFactor = (double) scaleSocket.getValue().get();
104+
final int minNeighbors = minNeighborsSocket.getValue().get().intValue();
105+
final Size minSize = minSizeSocket.getValue().get();
106+
final Size maxSize = maxSizeSocket.getValue().get();
107+
RectVector detections = new RectVector();
108+
classifier.detectMultiScale(image, detections, scaleFactor, minNeighbors, 0, minSize, maxSize);
109+
List<Rect> rects = new ArrayList<>();
110+
for (int i = 0; i < detections.size(); i++) {
111+
rects.add(detections.get(i));
112+
}
113+
output.setValue(new RectsReport(image, rects));
114+
}
115+
116+
}

‎core/src/main/java/edu/wpi/grip/core/operations/composite/RectsReport.java‎

Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,86 @@
1+
package edu.wpi.grip.core.operations.composite;
2+
3+
import edu.wpi.grip.core.operations.network.PublishValue;
4+
import edu.wpi.grip.core.operations.network.Publishable;
5+
import edu.wpi.grip.core.sockets.NoSocketTypeLabel;
6+
7+
import com.google.common.collect.ImmutableList;
8+
9+
import org.bytedeco.javacpp.opencv_core.Mat;
10+
import org.bytedeco.javacpp.opencv_core.Rect;
11+
12+
import java.util.ArrayList;
13+
import java.util.List;
14+
15+
/**
16+
* Contains the results of a rectangle detection operation.
17+
*/
18+
@NoSocketTypeLabel
19+
public class RectsReport implements Publishable {
20+
21+
private final Mat image;
22+
private final List<Rect> rectangles;
23+
24+
public static final RectsReport NIL = new RectsReport(new Mat(), new ArrayList<>());
25+
26+
public RectsReport(Mat image, List<Rect> rectangles) {
27+
this.image = image;
28+
this.rectangles = ImmutableList.copyOf(rectangles);
29+
}
30+
31+
/**
32+
* Gets the image the rectangles are for.
33+
*/
34+
public Mat getImage() {
35+
return image;
36+
}
37+
38+
/**
39+
* Gets the rectangles in this report.
40+
*/
41+
public List<Rect> getRectangles() {
42+
return rectangles;
43+
}
44+
45+
/**
46+
* An array of the coordinates of the X-values of the top-left corner of every rectangle.
47+
*/
48+
@PublishValue(key = "x", weight = 0)
49+
public double[] topLeftX() {
50+
return rectangles.stream()
51+
.mapToDouble(Rect::x)
52+
.toArray();
53+
}
54+
55+
56+
/**
57+
* An array of the coordinates of the Y-values of the top-left corner of every rectangle.
58+
*/
59+
@PublishValue(key = "y", weight = 1)
60+
public double[] topLeftY() {
61+
return rectangles.stream()
62+
.mapToDouble(Rect::y)
63+
.toArray();
64+
}
65+
66+
/**
67+
* An array of the widths of every rectangle.
68+
*/
69+
@PublishValue(key = "width", weight = 2)
70+
public double[] width() {
71+
return rectangles.stream()
72+
.mapToDouble(Rect::width)
73+
.toArray();
74+
}
75+
76+
/**
77+
* An array of the heights of every rectangle.
78+
*/
79+
@PublishValue(key = "height", weight = 3)
80+
public double[] height() {
81+
return rectangles.stream()
82+
.mapToDouble(Rect::height)
83+
.toArray();
84+
}
85+
86+
}

‎core/src/main/java/edu/wpi/grip/core/sockets/SocketHint.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,8 @@
22

33
import com.google.common.base.MoreObjects;
44

5+
import org.bytedeco.javacpp.opencv_objdetect.CascadeClassifier;
6+
57
import java.util.Arrays;
68
import java.util.List;
79
import java.util.NoSuchElementException;
@@ -122,6 +124,9 @@ public String getTypeLabel() {
122124
} else if (Mat.class.equals(type)) {
123125
// "Mats" represent images
124126
return "Image";
127+
} else if (CascadeClassifier.class.equals(type)) {
128+
// "CascadeClassifier" is too long and the name is already on the operation
129+
return "Classifier";
125130
} else {
126131
// For any other type, just use the name of the class
127132
return type.getSimpleName();

0 commit comments

Comments
 (0)