Skip to content

Fix bug with plugin security policy for GeoJSON output #63

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 9 commits into
base: master
Choose a base branch
from
Open

Fix bug with plugin security policy for GeoJSON output #63

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
646f440
add support for new json object list output format
fulups Sep 26, 2020
c5c2786
Merge pull request #2 from fulups/7.8.0-x
fulups Sep 26, 2020
3dbd3da
add support for geojson output format
fulups Dec 11, 2020
975bbf3
update some missing parameters in README
fulups Dec 11, 2020
e0575ef
Merge remote-tracking branch 'upstream/master'
fulups Dec 11, 2020
9497f1a
remove dirty debug message
fulups Dec 11, 2020
087bebb
fix bug with plugin security policy
fulups Dec 11, 2020
62100f1
update tests to correctly support variable docNumber
fulups Dec 11, 2020
7f64103
Merge remote-tracking branch 'upstream/master'
fulups Dec 11, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 14 additions & 8 deletions src/main/java/org/codelibs/elasticsearch/df/content/geojson/GeoJsonContent.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@
import java.io.IOException;
import java.io.OutputStreamWriter;
import java.io.Writer;
import java.security.AccessController;
import java.security.PrivilegedAction;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
Expand Down Expand Up @@ -118,7 +120,9 @@ protected OnLoadListener(final File outputFile, final ActionListener<Void> liste

@Override
public void onResponse(final SearchResponse response) {
final Gson gsonWriter = new GsonBuilder().create();
final Gson gsonWriter = AccessController.doPrivileged((PrivilegedAction<Gson>) () -> {
return new GsonBuilder().create();
});
final String scrollId = response.getScrollId();
final SearchHits hits = response.getHits();
final int size = hits.getHits().length;
Expand All @@ -137,20 +141,22 @@ public void onResponse(final SearchResponse response) {
firstLine = false;
}

final JsonElement propertiesJson = JsonParser.parseString(source);
final JsonElement propertiesJson = AccessController.doPrivileged((PrivilegedAction<JsonElement>) () -> {
return JsonParser.parseString(source);
});
String geometryType = "";

JsonArray geometryCoordinates = new JsonArray();
if (!geometryCoordinatesField.isEmpty()){
JsonElement jsonEltCoord = JsonUtils.getJsonElement(propertiesJson,geometryCoordinatesField);
final JsonElement jsonEltCoord = JsonUtils.getJsonElement(propertiesJson,geometryCoordinatesField);
if (jsonEltCoord !=null && !jsonEltCoord.isJsonNull()){
geometryCoordinates = jsonEltCoord.getAsJsonArray​();
if (!geometryKeepGeoInfo){
JsonUtils.removeJsonElement(propertiesJson,geometryCoordinatesField);
}
}
if (!geometryTypeField.isEmpty()){
JsonElement jsonEltType = JsonUtils.getJsonElement(propertiesJson,geometryTypeField);
final JsonElement jsonEltType = JsonUtils.getJsonElement(propertiesJson,geometryTypeField);
if (jsonEltType !=null && !jsonEltType.isJsonNull()){
geometryType = jsonEltType.getAsString();
if (!geometryKeepGeoInfo){
Expand All @@ -160,8 +166,8 @@ public void onResponse(final SearchResponse response) {
}
}else{
if (!geometryCoordinatesLonField.isEmpty() && !geometryCoordinatesLatField.isEmpty()){
JsonElement jsonEltLon = JsonUtils.getJsonElement(propertiesJson,geometryCoordinatesLonField);
JsonElement jsonEltLat = JsonUtils.getJsonElement(propertiesJson,geometryCoordinatesLatField);
final JsonElement jsonEltLon = JsonUtils.getJsonElement(propertiesJson,geometryCoordinatesLonField);
final JsonElement jsonEltLat = JsonUtils.getJsonElement(propertiesJson,geometryCoordinatesLatField);
if (jsonEltLon !=null && !jsonEltLon.isJsonNull() && jsonEltLat !=null && !jsonEltLat.isJsonNull()){
geometryCoordinates.add(jsonEltLon.getAsNumber());
geometryCoordinates.add(jsonEltLat.getAsNumber());
Expand All @@ -172,7 +178,7 @@ public void onResponse(final SearchResponse response) {
}
}
if (!geometryCoordinatesAltField.isEmpty()){
JsonElement jsonElt = JsonUtils.getJsonElement(propertiesJson,geometryCoordinatesAltField);
final JsonElement jsonElt = JsonUtils.getJsonElement(propertiesJson,geometryCoordinatesAltField);
if (jsonElt !=null && !jsonElt.isJsonNull()){
geometryCoordinates.add(jsonElt.getAsNumber());
if (!geometryKeepGeoInfo) {
Expand All @@ -186,7 +192,7 @@ public void onResponse(final SearchResponse response) {
for (String excludeField : excludeFields) {
JsonUtils.removeJsonElement(propertiesJson,excludeField);
}

JsonObject geometryObject = new JsonObject();
geometryObject.addProperty("type", geometryType);
geometryObject.add("coordinates", geometryCoordinates);
Expand Down
1 change: 1 addition & 0 deletions src/main/plugin-metadata/plugin-security.policy
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
grant {
permission java.lang.RuntimePermission "getClassLoader";
permission java.lang.RuntimePermission "accessDeclaredMembers";
};
74 changes: 46 additions & 28 deletions src/test/java/org/codelibs/elasticsearch/df/DataFormatPluginTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,8 @@ public class DataFormatPluginTest {
private final Map<String, String> paramsGeoJson = new HashMap<>();

static {
// Doc number used for test
// -> max 9990 because elastic query size limited below 10000
docNumber = 20;

csvTempFile = createTempFile("csvtest", ".csv");
Expand Down Expand Up @@ -168,14 +170,15 @@ public void dumpCsvWithQuery() throws IOException {
assertLineContains(lines[0], "\"aaa\"", "\"bbb\"", "\"ccc\"", "\"eee.fff\"", "\"eee.ggg\"");
assertLineContains(lines[1], "\"1\"");
}

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

please remove spaces.

// Download 10 docs as CSV
clearParams();
prepareParams();
paramsCsv.put("q", "*:*");
paramsCsv.put("from", "5");
try (CurlResponse response = createRequest(node, path, paramsCsv).execute()) {
assertEquals(16, response.getContentAsString().split("\n").length);
try (CurlResponse response = createRequest(node, path, paramsCsv)
.param("size", Integer.toString(docNumber)).execute()) {
assertEquals(docNumber - 4, response.getContentAsString().split("\n").length);
}

// Download all the docs from the 5th as CSV
Expand All @@ -193,7 +196,9 @@ public void dumpCsvWithQuery() throws IOException {
// Download All as CSV with Query and from
try (CurlResponse curlResponse = EcrCurl.get(node, "/dataset0/_data")
.header("Content-Type", "application/json")
.param("format", "csv").body(queryWithFrom).execute()) {
.param("format", "csv")
.param("size", Integer.toString(docNumber))
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

size is in queryWithFrom.

.body(queryWithFrom).execute()) {
final String content = curlResponse.getContentAsString();
final String[] lines = content.split("\n");
assertEquals(docNumber - 10 + 1, lines.length);
Expand All @@ -204,6 +209,7 @@ public void dumpCsvWithQuery() throws IOException {
.header("Content-Type", "application/json")
.param("format", "csv").param("source", queryWithFrom)
.param("source_content_type", "application/json")
.param("size", String.valueOf(docNumber))
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

size is in queryWithFrom.

.execute()) {
final String content = curlResponse.getContentAsString();
final String[] lines = content.split("\n");
Expand All @@ -214,6 +220,7 @@ public void dumpCsvWithQuery() throws IOException {
try (CurlResponse curlResponse = EcrCurl.get(node, "/dataset0/_data")
.header("Content-Type", "application/json")
.param("search_type", "query_then_fetch").param("format", "csv")
.param("size", String.valueOf(docNumber))
.execute()) {
final String content = curlResponse.getContentAsString();
final String[] lines = content.split("\n");
Expand Down Expand Up @@ -299,7 +306,9 @@ public void dumpExcel() throws IOException {
// Download All as Excel with search_type
try (CurlResponse curlResponse = EcrCurl.get(node, "/dataset0/_data")
.header("Content-Type", "application/json")
.param("search_type", "query_then_fetch").param("format", "xls")
.param("search_type", "query_then_fetch")
.param("format", "xls")
.param("size", Integer.toString(docNumber))
.execute()) {
try (InputStream is = curlResponse.getContentAsStream()) {
final POIFSFileSystem fs = new POIFSFileSystem(is);
Expand Down Expand Up @@ -331,7 +340,8 @@ public void dumpJson() throws IOException {
// Download All as JSON
try (CurlResponse curlResponse = EcrCurl.get(node, "/dataset0/_data")
.header("Content-Type", "application/json")
.param("format", "json").execute()) {
.param("format", "json")
.param("size", Integer.toString(docNumber)).execute()) {
final String content = curlResponse.getContentAsString();
final String[] lines = content.split("\n");
assertEquals(docNumber * 2, lines.length);
Expand All @@ -343,6 +353,7 @@ public void dumpJson() throws IOException {
try (CurlResponse curlResponse = EcrCurl.get(node, "/dataset0/_data")
.header("Content-Type", "application/json")
.param("format", "json").param("bulk.index", "dataset02")
.param("size", Integer.toString(docNumber))
.execute()) {
final String content = curlResponse.getContentAsString();
final String[] lines = content.split("\n");
Expand All @@ -366,15 +377,6 @@ public void dumpJson() throws IOException {
assertTrue(lines[1].startsWith("{\"aaa\":\"test 1\","));
}

// Download 10 docs as JSON
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why did you remove this code?

try (CurlResponse curlResponse = EcrCurl.get(node, "/dataset0/_data")
.header("Content-Type", "application/json").param("q", "*:*")
.param("format", "json").param("from", "5").execute()) {
final String content = curlResponse.getContentAsString();
final String[] lines = content.split("\n");
assertEquals(30, lines.length);
}

// Download all the docs from the 5th as JSON
try (CurlResponse curlResponse = EcrCurl.get(node, "/dataset0/_data")
.header("Content-Type", "application/json").param("q", "*:*")
Expand All @@ -390,7 +392,9 @@ public void dumpJson() throws IOException {
// Download All as JSON with Query and from
try (CurlResponse curlResponse = EcrCurl.get(node, "/dataset0/_data")
.header("Content-Type", "application/json")
.param("format", "json").body(queryWithFrom).execute()) {
.param("format", "json")
.param("size", String.valueOf(docNumber))
.body(queryWithFrom).execute()) {
final String content = curlResponse.getContentAsString();
final String[] lines = content.split("\n");
assertEquals((docNumber - 5) * 2, lines.length);
Expand All @@ -401,6 +405,7 @@ public void dumpJson() throws IOException {
.header("Content-Type", "application/json")
.param("format", "json").param("source", queryWithFrom)
.param("source_content_type", "application/json")
.param("size", String.valueOf(docNumber))
.execute()) {
final String content = curlResponse.getContentAsString();
final String[] lines = content.split("\n");
Expand All @@ -411,7 +416,8 @@ public void dumpJson() throws IOException {
try (CurlResponse curlResponse = EcrCurl.get(node, "/dataset0/_data")
.header("Content-Type", "application/json")
.param("search_type", "query_then_fetch")
.param("format", "json").execute()) {
.param("format", "json")
.param("size", String.valueOf(docNumber)).execute()) {
final String content = curlResponse.getContentAsString();
final String[] lines = content.split("\n");
assertEquals(docNumber * 2, lines.length);
Expand Down Expand Up @@ -440,7 +446,7 @@ public void dumpJsonList() throws IOException {
// Download All as JSON
try (CurlResponse curlResponse = EcrCurl.get(node, "/dataset0/_data")
.header("Content-Type", "application/json")
.param("format", "jsonlist").execute()) {
.param("format", "jsonlist").param("size", Integer.toString(docNumber)).execute()) {
final String content = curlResponse.getContentAsString();
final String[] lines = content.split("\n");
assertEquals(docNumber + 2, lines.length);
Expand All @@ -467,10 +473,11 @@ public void dumpJsonList() throws IOException {
// Download 10 docs as JSON
try (CurlResponse curlResponse = EcrCurl.get(node, "/dataset0/_data")
.header("Content-Type", "application/json").param("q", "*:*")
.param("format", "jsonlist").param("from", "5").execute()) {
.param("format", "jsonlist").param("from", "5")
.param("size", Integer.toString(docNumber)).execute()) {
final String content = curlResponse.getContentAsString();
final String[] lines = content.split("\n");
assertEquals(15 + 2, lines.length);
assertEquals(docNumber - 5 + 2, lines.length);
}

// Download all the docs from the 5th as JSON
Expand All @@ -488,7 +495,9 @@ public void dumpJsonList() throws IOException {
// Download All as JSON with Query and from
try (CurlResponse curlResponse = EcrCurl.get(node, "/dataset0/_data")
.header("Content-Type", "application/json")
.param("format", "jsonlist").body(queryWithFrom).execute()) {
.param("format", "jsonlist")
.param("size", String.valueOf(docNumber))
.body(queryWithFrom).execute()) {
final String content = curlResponse.getContentAsString();
final String[] lines = content.split("\n");
assertEquals((docNumber - 5) + 2, lines.length);
Expand All @@ -499,7 +508,7 @@ public void dumpJsonList() throws IOException {
.header("Content-Type", "application/json")
.param("format", "jsonlist").param("source", queryWithFrom)
.param("source_content_type", "application/json")
.execute()) {
.param("size", String.valueOf(docNumber)).execute()) {
final String content = curlResponse.getContentAsString();
final String[] lines = content.split("\n");
assertEquals((docNumber - 5) + 2, lines.length);
Expand All @@ -509,7 +518,8 @@ public void dumpJsonList() throws IOException {
try (CurlResponse curlResponse = EcrCurl.get(node, "/dataset0/_data")
.header("Content-Type", "application/json")
.param("search_type", "query_then_fetch")
.param("format", "jsonlist").execute()) {
.param("format", "jsonlist")
.param("size", String.valueOf(docNumber)).execute()) {
final String content = curlResponse.getContentAsString();
final String[] lines = content.split("\n");
assertEquals(docNumber + 2, lines.length);
Expand Down Expand Up @@ -540,7 +550,7 @@ public void dumpGeoJson() throws IOException {
// default call
try (CurlResponse curlResponse = EcrCurl.get(node, "/dataset1/_data")
.header("Content-Type", "application/json")
.param("format", "geojson").execute()) {
.param("format", "geojson").param("size", Integer.toString(docNumber)).execute()) {
final String content = curlResponse.getContentAsString();
final String[] lines = content.split("\n");
assertEquals(docNumber + 2, lines.length);
Expand Down Expand Up @@ -658,7 +668,9 @@ public void dumpSizeLimit() throws IOException {
// Default
try (CurlResponse curlResponse = EcrCurl.get(node, "/dataset0/_data")
.header("Content-Type", "application/json")
.param("format", "csv").execute()) {
.param("format", "csv")
.param("size", Integer.toString(docNumber))
.execute()) {
final String content = curlResponse.getContentAsString();
final String[] lines = content.split("\n");
assertEquals(docNumber + 1, lines.length);
Expand All @@ -667,7 +679,10 @@ public void dumpSizeLimit() throws IOException {
// 50%
try (CurlResponse curlResponse = EcrCurl.get(node, "/dataset0/_data")
.header("Content-Type", "application/json")
.param("format", "csv").param("limit", "50%").execute()) {
.param("format", "csv")
.param("size", Integer.toString(docNumber))
.param("limit", "50%")
.execute()) {
final String content = curlResponse.getContentAsString();
final String[] lines = content.split("\n");
assertEquals(docNumber + 1, lines.length);
Expand All @@ -676,7 +691,10 @@ public void dumpSizeLimit() throws IOException {
//0%
try (CurlResponse curlResponse = EcrCurl.get(node, "/dataset0/_data")
.header("Content-Type", "application/json")
.param("format", "csv").param("limit", "0").execute()) {
.param("format", "csv")
.param("size", Integer.toString(docNumber))
.param("limit", "0")
.execute()) {
assertEquals(500, curlResponse.getHttpStatusCode());
}
}
Expand Down Expand Up @@ -762,7 +780,7 @@ private static File createTempFile(String prefix, String suffix) {
}

private CurlRequest createRequest(Node node, String path, Map<String, String> params) {
CurlRequest request = EcrCurl.get(node, path).header("Content-Type", "application/json");
CurlRequest request = EcrCurl.get(node, path).header("Content-Type", "application/json").param("size", Integer.toString(docNumber));
for (final Map.Entry<String, String> entry : params.entrySet()) {
request.param(entry.getKey(), entry.getValue());
}
Expand Down