Skip to content
This repository was archived by the owner on May 12, 2021. It is now read-only.

Commit 52eb306

Browse files
shimamotoshimamoto
authored
[PIO-203] Fixes pio status warnings in ES storage (#507)
1 parent aa79f7c commit 52eb306

File tree

8 files changed

+31
-70
lines changed

8 files changed

+31
-70
lines changed

‎storage/elasticsearch/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESAccessKeys.scala

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -42,12 +42,9 @@ class ESAccessKeys(client: RestClient, config: StorageClientConfig, index: Strin
4242
private val estype = "accesskeys"
4343
private val internalIndex = index + "_" + estype
4444

45-
ESUtils.createIndex(client, internalIndex,
46-
ESUtils.getNumberOfShards(config, internalIndex.toUpperCase),
47-
ESUtils.getNumberOfReplicas(config, internalIndex.toUpperCase))
45+
ESUtils.createIndex(client, internalIndex)
4846
val mappingJson =
4947
(estype ->
50-
("_all" -> ("enabled" -> false)) ~
5148
("properties" ->
5249
("key" -> ("type" -> "keyword")) ~
5350
("events" -> ("type" -> "keyword"))))

‎storage/elasticsearch/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESApps.scala

Lines changed: 2 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -40,16 +40,12 @@ class ESApps(client: RestClient, config: StorageClientConfig, index: String)
4040
extends Apps with Logging {
4141
implicit val formats = DefaultFormats.lossless
4242
private val estype = "apps"
43+
private val seq = new ESSequences(client, config, index)
4344
private val internalIndex = index + "_" + estype
4445

45-
private val seq = new ESSequences(client, config, internalIndex)
46-
47-
ESUtils.createIndex(client, internalIndex,
48-
ESUtils.getNumberOfShards(config, internalIndex.toUpperCase),
49-
ESUtils.getNumberOfReplicas(config, internalIndex.toUpperCase))
46+
ESUtils.createIndex(client, internalIndex)
5047
val mappingJson =
5148
(estype ->
52-
("_all" -> ("enabled" -> false)) ~
5349
("properties" ->
5450
("id" -> ("type" -> "keyword")) ~
5551
("name" -> ("type" -> "keyword"))))

‎storage/elasticsearch/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESChannels.scala

Lines changed: 2 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -39,15 +39,12 @@ class ESChannels(client: RestClient, config: StorageClientConfig, index: String)
3939
extends Channels with Logging {
4040
implicit val formats = DefaultFormats.lossless
4141
private val estype = "channels"
42-
private val seq = new ESSequences(client, config, internalIndex)
42+
private val seq = new ESSequences(client, config, index)
4343
private val internalIndex = index + "_" + estype
4444

45-
ESUtils.createIndex(client, internalIndex,
46-
ESUtils.getNumberOfShards(config, internalIndex.toUpperCase),
47-
ESUtils.getNumberOfReplicas(config, internalIndex.toUpperCase))
45+
ESUtils.createIndex(client, internalIndex)
4846
val mappingJson =
4947
(estype ->
50-
("_all" -> ("enabled" -> false)) ~
5148
("properties" ->
5249
("name" -> ("type" -> "keyword"))))
5350
ESUtils.createMapping(client, internalIndex, estype, compact(render(mappingJson)))

‎storage/elasticsearch/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESEngineInstances.scala

Lines changed: 20 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -40,13 +40,11 @@ class ESEngineInstances(client: RestClient, config: StorageClientConfig, index:
4040
extends EngineInstances with Logging {
4141
implicit val formats = DefaultFormats + new EngineInstanceSerializer
4242
private val estype = "engine_instances"
43-
44-
ESUtils.createIndex(client, index,
45-
ESUtils.getNumberOfShards(config, index.toUpperCase),
46-
ESUtils.getNumberOfReplicas(config, index.toUpperCase))
43+
private val internalIndex = index + "_" + estype
44+
45+
ESUtils.createIndex(client, internalIndex)
4746
val mappingJson =
4847
(estype ->
49-
("_all" -> ("enabled" -> false)) ~
5048
("properties" ->
5149
("status" -> ("type" -> "keyword")) ~
5250
("startTime" -> ("type" -> "date")) ~
@@ -61,7 +59,7 @@ class ESEngineInstances(client: RestClient, config: StorageClientConfig, index:
6159
("algorithmsParams" -> ("type" -> "keyword")) ~
6260
("servingParams" -> ("type" -> "keyword"))
6361
))
64-
ESUtils.createMapping(client, index, estype, compact(render(mappingJson)))
62+
ESUtils.createMapping(client, internalIndex, estype, compact(render(mappingJson)))
6563

6664
def insert(i: EngineInstance): String = {
6765
val id = i.id match {
@@ -86,7 +84,7 @@ class ESEngineInstances(client: RestClient, config: StorageClientConfig, index:
8684
val entity = new NStringEntity("{}", ContentType.APPLICATION_JSON)
8785
val response = client.performRequest(
8886
"POST",
89-
s"/$index/$estype/",
87+
s"/$internalIndex/$estype/",
9088
Map("refresh" -> "true").asJava,
9189
entity)
9290
val jsonResponse = parse(EntityUtils.toString(response.getEntity))
@@ -95,12 +93,12 @@ class ESEngineInstances(client: RestClient, config: StorageClientConfig, index:
9593
case "created" =>
9694
Some((jsonResponse \ "_id").extract[String])
9795
case _ =>
98-
error(s"[$result] Failed to create $index/$estype")
96+
error(s"[$result] Failed to create $internalIndex/$estype")
9997
None
10098
}
10199
} catch {
102100
case e: IOException =>
103-
error(s"Failed to create $index/$estype", e)
101+
error(s"Failed to create $internalIndex/$estype", e)
104102
None
105103
}
106104
}
@@ -109,7 +107,7 @@ class ESEngineInstances(client: RestClient, config: StorageClientConfig, index:
109107
try {
110108
val response = client.performRequest(
111109
"GET",
112-
s"/$index/$estype/$id",
110+
s"/$internalIndex/$estype/$id",
113111
Map.empty[String, String].asJava)
114112
val jsonResponse = parse(EntityUtils.toString(response.getEntity))
115113
(jsonResponse \ "found").extract[Boolean] match {
@@ -123,11 +121,11 @@ class ESEngineInstances(client: RestClient, config: StorageClientConfig, index:
123121
e.getResponse.getStatusLine.getStatusCode match {
124122
case 404 => None
125123
case _ =>
126-
error(s"Failed to access to /$index/$estype/$id", e)
124+
error(s"Failed to access to /$internalIndex/$estype/$id", e)
127125
None
128126
}
129127
case e: IOException =>
130-
error(s"Failed to access to /$index/$estype/$id", e)
128+
error(s"Failed to access to /$internalIndex/$estype/$id", e)
131129
None
132130
}
133131
}
@@ -137,10 +135,10 @@ class ESEngineInstances(client: RestClient, config: StorageClientConfig, index:
137135
val json =
138136
("query" ->
139137
("match_all" -> List.empty))
140-
ESUtils.getAll[EngineInstance](client, index, estype, compact(render(json)))
138+
ESUtils.getAll[EngineInstance](client, internalIndex, estype, compact(render(json)))
141139
} catch {
142140
case e: IOException =>
143-
error("Failed to access to /$index/$estype/_search", e)
141+
error(s"Failed to access to /$internalIndex/$estype/_search", e)
144142
Nil
145143
}
146144
}
@@ -165,10 +163,10 @@ class ESEngineInstances(client: RestClient, config: StorageClientConfig, index:
165163
("sort" -> List(
166164
("startTime" ->
167165
("order" -> "desc"))))
168-
ESUtils.getAll[EngineInstance](client, index, estype, compact(render(json)))
166+
ESUtils.getAll[EngineInstance](client, internalIndex, estype, compact(render(json)))
169167
} catch {
170168
case e: IOException =>
171-
error(s"Failed to access to /$index/$estype/_search", e)
169+
error(s"Failed to access to /$internalIndex/$estype/_search", e)
172170
Nil
173171
}
174172
}
@@ -188,7 +186,7 @@ class ESEngineInstances(client: RestClient, config: StorageClientConfig, index:
188186
val entity = new NStringEntity(write(i), ContentType.APPLICATION_JSON)
189187
val response = client.performRequest(
190188
"POST",
191-
s"/$index/$estype/$id",
189+
s"/$internalIndex/$estype/$id",
192190
Map("refresh" -> "true").asJava,
193191
entity)
194192
val jsonResponse = parse(EntityUtils.toString(response.getEntity))
@@ -197,30 +195,30 @@ class ESEngineInstances(client: RestClient, config: StorageClientConfig, index:
197195
case "created" =>
198196
case "updated" =>
199197
case _ =>
200-
error(s"[$result] Failed to update $index/$estype/$id")
198+
error(s"[$result] Failed to update $internalIndex/$estype/$id")
201199
}
202200
} catch {
203201
case e: IOException =>
204-
error(s"Failed to update $index/$estype/$id", e)
202+
error(s"Failed to update $internalIndex/$estype/$id", e)
205203
}
206204
}
207205

208206
def delete(id: String): Unit = {
209207
try {
210208
val response = client.performRequest(
211209
"DELETE",
212-
s"/$index/$estype/$id",
210+
s"/$internalIndex/$estype/$id",
213211
Map("refresh" -> "true").asJava)
214212
val json = parse(EntityUtils.toString(response.getEntity))
215213
val result = (json \ "result").extract[String]
216214
result match {
217215
case "deleted" =>
218216
case _ =>
219-
error(s"[$result] Failed to update $index/$estype/$id")
217+
error(s"[$result] Failed to update $internalIndex/$estype/$id")
220218
}
221219
} catch {
222220
case e: IOException =>
223-
error(s"Failed to update $index/$estype/$id", e)
221+
error(s"Failed to update $internalIndex/$estype/$id", e)
224222
}
225223
}
226224
}

‎storage/elasticsearch/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESEvaluationInstances.scala

Lines changed: 2 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,6 @@ import org.apache.predictionio.data.storage.EvaluationInstance
2828
import org.apache.predictionio.data.storage.EvaluationInstanceSerializer
2929
import org.apache.predictionio.data.storage.EvaluationInstances
3030
import org.apache.predictionio.data.storage.StorageClientConfig
31-
import org.apache.predictionio.data.storage.StorageClientException
3231
import org.elasticsearch.client.{ResponseException, RestClient}
3332
import org.json4s._
3433
import org.json4s.JsonDSL._
@@ -41,15 +40,12 @@ class ESEvaluationInstances(client: RestClient, config: StorageClientConfig, ind
4140
extends EvaluationInstances with Logging {
4241
implicit val formats = DefaultFormats + new EvaluationInstanceSerializer
4342
private val estype = "evaluation_instances"
44-
private val seq = new ESSequences(client, config, internalIndex)
43+
private val seq = new ESSequences(client, config, index)
4544
private val internalIndex = index + "_" + estype
4645

47-
ESUtils.createIndex(client, internalIndex,
48-
ESUtils.getNumberOfShards(config, internalIndex.toUpperCase),
49-
ESUtils.getNumberOfReplicas(config, internalIndex.toUpperCase))
46+
ESUtils.createIndex(client, internalIndex)
5047
val mappingJson =
5148
(estype ->
52-
("_all" -> ("enabled" -> false)) ~
5349
("properties" ->
5450
("status" -> ("type" -> "keyword")) ~
5551
("startTime" -> ("type" -> "date")) ~

‎storage/elasticsearch/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESLEvents.scala

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -53,12 +53,9 @@ class ESLEvents(val client: RestClient, config: StorageClientConfig, val baseInd
5353
override def init(appId: Int, channelId: Option[Int] = None): Boolean = {
5454
val estype = getEsType(appId, channelId)
5555
val index = baseIndex + "_" + estype
56-
ESUtils.createIndex(client, index,
57-
ESUtils.getNumberOfShards(config, index.toUpperCase),
58-
ESUtils.getNumberOfReplicas(config, index.toUpperCase))
56+
ESUtils.createIndex(client, index)
5957
val json =
6058
(estype ->
61-
("_all" -> ("enabled" -> false)) ~
6259
("properties" ->
6360
("name" -> ("type" -> "keyword")) ~
6461
("eventId" -> ("type" -> "keyword")) ~

‎storage/elasticsearch/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESSequences.scala

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,6 @@ import java.io.IOException
2121

2222
import scala.collection.JavaConverters._
2323

24-
import org.apache.http.Header
2524
import org.apache.http.entity.ContentType
2625
import org.apache.http.nio.entity.NStringEntity
2726
import org.apache.http.util.EntityUtils
@@ -40,12 +39,9 @@ class ESSequences(client: RestClient, config: StorageClientConfig, index: String
4039
private val estype = "sequences"
4140
private val internalIndex = index + "_" + estype
4241

43-
ESUtils.createIndex(client, internalIndex,
44-
ESUtils.getNumberOfShards(config, internalIndex.toUpperCase),
45-
ESUtils.getNumberOfReplicas(config, internalIndex.toUpperCase))
42+
ESUtils.createIndex(client, internalIndex)
4643
val mappingJson =
4744
(estype ->
48-
("_all" -> ("enabled" -> false)) ~
4945
("properties" ->
5046
("n" -> ("enabled" -> false))))
5147
ESUtils.createMapping(client, internalIndex, estype, compact(render(mappingJson)))

‎storage/elasticsearch/src/main/scala/org/apache/predictionio/data/storage/elasticsearch/ESUtils.scala

Lines changed: 2 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,6 @@ import scala.collection.JavaConversions._
2121
import scala.collection.JavaConverters._
2222

2323
import org.apache.http.entity.ContentType
24-
import org.apache.http.entity.StringEntity
2524
import org.apache.http.nio.entity.NStringEntity
2625
import org.elasticsearch.client.RestClient
2726
import org.json4s._
@@ -165,23 +164,16 @@ object ESUtils {
165164

166165
def createIndex(
167166
client: RestClient,
168-
index: String,
169-
numberOfShards: Option[Int],
170-
numberOfReplicas: Option[Int]): Unit = {
167+
index: String): Unit = {
171168
client.performRequest(
172169
"HEAD",
173170
s"/$index",
174171
Map.empty[String, String].asJava).getStatusLine.getStatusCode match {
175172
case 404 =>
176-
val json = ("settings" ->
177-
("number_of_shards" -> numberOfShards) ~
178-
("number_of_replicas" -> numberOfReplicas))
179-
val entity = new NStringEntity(compact(render(json)), ContentType.APPLICATION_JSON)
180173
client.performRequest(
181174
"PUT",
182175
s"/$index",
183-
Map.empty[String, String].asJava,
184-
entity)
176+
Map.empty[String, String].asJava)
185177
case 200 =>
186178
case _ =>
187179
throw new IllegalStateException(s"/$index is invalid.")
@@ -269,14 +261,6 @@ object ESUtils {
269261
(hosts, ports, schemes).zipped.map((h, p, s) => new HttpHost(h, p, s))
270262
}
271263

272-
def getNumberOfShards(config: StorageClientConfig, index: String): Option[Int] = {
273-
config.properties.get(s"${index}_NUM_OF_SHARDS").map(_.toInt)
274-
}
275-
276-
def getNumberOfReplicas(config: StorageClientConfig, index: String): Option[Int] = {
277-
config.properties.get(s"${index}_NUM_OF_REPLICAS").map(_.toInt)
278-
}
279-
280264
def getEventDataRefresh(config: StorageClientConfig): String = {
281265
config.properties.getOrElse("EVENTDATA_REFRESH", "true")
282266
}

0 commit comments

Comments
 (0)