Reading point geometry from HBASE tables in EMR emr-5.14.0 configuration. Hbase 1.4.2, Hue 4.1.0, JupyterHub 0.8.1, Livy 0.4.0, Oozie 4.3.0, Spark 2.3.0, Zeppelin 0.7.3 Table has > 5 Million points. Running code in zeppelin notebook the followng:
- Configure OPTIONS
conf = geomesa_pyspark.configure(packages='geomesa_pyspark', 'pytz') hbase_params = {"hbase.catalog":"geomesa.mover"} INCREMENT_DAYS = 1 POINTS_NAME = "mover_points" sparkSession = ( SparkSession .builder .config(conf=conf) .getOrCreate() ) points_df = sparkSession.read.format("geomesa").options(**hbase_params).option("geomesa.feature",
- This lets you use SparkSQL string queries (example down below)
points_df.createOrReplaceTempView("points_df")
points_df.count() Receive exception stack trace below. Do other operations like Spark SQL Queries and have no errors. Was able to run this in 2.0.2 without a problem. Traceback (most recent call last): File "/tmp/zeppelin_pyspark-3601066725639987471.py", line 360, in <module> exec(code, _zcUserQueryNameSpace) File "<stdin>", line 2, in <module> File "/usr/lib/spark/python/pyspark/sql/dataframe.py", line 455, in count return int(self._jdf.count()) File "/usr/lib/spark/python/lib/py4j-0.10.6-src.zip/py4j/java_gateway.py", line 1160, in _call_ answer, self.gateway_client, self.target_id, self.name) File "/usr/lib/spark/python/pyspark/sql/utils.py", line 63, in deco return f(*a, **kw) File "/usr/lib/spark/python/lib/py4j-0.10.6-src.zip/py4j/protocol.py", line 320, in get_return_value format(target_id, ".", name), value) py4j.protocol.Py4JJavaError: An error occurred while calling o264.count. : org.apache.spark.sql.catalyst.errors.package$TreeNodeException: execute, tree: Exchange SinglePartition +- *(1) HashAggregate(keys=[], functions=partial_count(1), output=count#1084L) +- *(1) Scan GeoMesaRelation(org.apache.spark.sql.SQLContext@634c3184,SimpleFeatureTypeImpl mover_points identified extends Feature(data_src:data_src,distance_km:distance_km,docked_port_id:docked_port_id,duration_sec:duration_sec,geom:geom,heading_rad:heading_rad,imsi:imsi,init_time:init_time,meta:meta,next_port_id:next_port_id,next_port_arrival:next_port_arrival,prev_point:prev_point,prev_port_departure:prev_port_departure,prev_port_id:prev_port_id,prev_time:prev_time,segment_id:segment_id,speed_kph:speed_kph),StructType(StructField(_fid_,StringType,false), StructField(data_src,StringType,true), StructField(distance_km,DoubleType,true), StructField(docked_port_id,StringType,true), StructField(duration_sec,IntegerType,true), StructField(geom,org.apache.spark.sql.jts.PointUDT@6a7d3f2b,true), StructField(heading_rad,DoubleType,true), StructField(imsi,StringType,true), StructField(init_time,TimestampType,true), StructField(meta,StringType,true), StructField(next_port_id,StringType,true), StructField(next_port_arrival,TimestampType,true), StructField(prev_point,org.apache.spark.sql.jts.PointUDT@6a7d3f2b,true), StructField(prev_port_departure,TimestampType,true), StructField(prev_port_id,StringType,true), StructField(prev_time,TimestampType,true), StructField(segment_id,StringType,true), StructField(speed_kph,DoubleType,true)),Map(geomesa.feature -> mover_points, hbase.catalog -> geomesa.mover),Filter.INCLUDE,None,null,null,null,null) [] PushedFilters: [], ReadSchema: struct<> at org.apache.spark.sql.catalyst.errors.package$.attachTree(package.scala:56) at org.apache.spark.sql.execution.exchange.ShuffleExchangeExec.doExecute(ShuffleExchangeExec.scala:119) at org.apache.spark.sql.execution.SparkPlan$$anonfun$execute$1.apply(SparkPlan.scala:131) at org.apache.spark.sql.execution.SparkPlan$$anonfun$execute$1.apply(SparkPlan.scala:127) at org.apache.spark.sql.execution.SparkPlan$$anonfun$executeQuery$1.apply(SparkPlan.scala:155) at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151) at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:152) at org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:127) at org.apache.spark.sql.execution.InputAdapter.inputRDDs(WholeStageCodegenExec.scala:371) at org.apache.spark.sql.execution.aggregate.HashAggregateExec.inputRDDs(HashAggregateExec.scala:150) at org.apache.spark.sql.execution.WholeStageCodegenExec.doExecute(WholeStageCodegenExec.scala:605) at org.apache.spark.sql.execution.SparkPlan$$anonfun$execute$1.apply(SparkPlan.scala:131) at org.apache.spark.sql.execution.SparkPlan$$anonfun$execute$1.apply(SparkPlan.scala:127) at org.apache.spark.sql.execution.SparkPlan$$anonfun$executeQuery$1.apply(SparkPlan.scala:155) at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151) at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:152) at org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:127) at org.apache.spark.sql.execution.SparkPlan.getByteArrayRdd(SparkPlan.scala:247) at org.apache.spark.sql.execution.SparkPlan.executeCollect(SparkPlan.scala:294) at org.apache.spark.sql.Dataset$$anonfun$count$1.apply(Dataset.scala:2770) at org.apache.spark.sql.Dataset$$anonfun$count$1.apply(Dataset.scala:2769) at org.apache.spark.sql.Dataset$$anonfun$52.apply(Dataset.scala:3253) at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:77) at org.apache.spark.sql.Dataset.withAction(Dataset.scala:3252) at org.apache.spark.sql.Dataset.count(Dataset.scala:2769) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) at py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) at py4j.Gateway.invoke(Gateway.java:282) at py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) at py4j.commands.CallCommand.execute(CallCommand.java:79) at py4j.GatewayConnection.run(GatewayConnection.java:214) at java.lang.Thread.run(Thread.java:748) Caused by: java.lang.RuntimeException: Failed to convert Scan : CgYKAWQSAWQSQAoac2Nhbi5hdHRyaWJ1dGVzLnRhYmxlLm5hbWUSImdlb21lc2EubW92ZXJfbW92 ZXJfNWZwb2ludHNfejNfdjIqhwUKKW9yZy5hcGFjaGUuaGFkb29wLmhiYXNlLmZpbHRlci5GaWx0 ZXJMaXN0EtkECAESQAoyb3JnLmFwYWNoZS5oYWRvb3AuaGJhc2UuZmlsdGVyLk11bHRpUm93UmFu Z2VGaWx0ZXISCgoICgAQARoAIAASkgQKOW9yZy5sb2NhdGlvbnRlY2guZ2VvbWVzYS5oYmFzZS5m aWx0ZXJzLkNxbFRyYW5zZm9ybUZpbHRlchLUAwAAAcRkYXRhX3NyYzpTdHJpbmcsZGlzdGFuY2Vf a206RG91YmxlLGRvY2tlZF9wb3J0X2lkOlN0cmluZyxkdXJhdGlvbl9zZWM6SW50ZWdlciwqZ2Vv bTpQb2ludDpzcmlkPTQzMjYsaGVhZGluZ19yYWQ6RG91YmxlLGltc2k6U3RyaW5nOmluZGV4PXRy dWUsaW5pdF90aW1lOkRhdGUsbWV0YTpTdHJpbmcsbmV4dF9wb3J0X2lkOlN0cmluZyxuZXh0X3Bv cnRfYXJyaXZhbDpEYXRlLHByZXZfcG9pbnQ6UG9pbnQ6c3JpZD00MzI2LHByZXZfcG9ydF9kZXBh cnR1cmU6RGF0ZSxwcmV2X3BvcnRfaWQ6U3RyaW5nLHByZXZfdGltZTpEYXRlLHNlZ21lbnRfaWQ6 U3RyaW5nOmluZGV4PXRydWUsc3BlZWRfa3BoOkRvdWJsZTtnZW9tZXNhLmluZGV4LmR0Zz0naW5p dF90aW1lJyxnZW9tZXNhLnRhYmxlLnNoYXJpbmc9J2ZhbHNlJyxnZW9tZXNhLmluZGljZXM9J3oz OjI6Myx6MjoyOjMsaWQ6MTozLGF0dHI6NTozJwAAAAAAAAAAAAAAADgBQAGIAegH to string at org.apache.hadoop.hbase.mapreduce.MultiTableInputFormat.setConf(MultiTableInputFormat.java:101) at org.locationtech.geomesa.hbase.jobs.GeoMesaHBaseInputFormat.init(GeoMesaHBaseInputFormat.scala:39) at org.locationtech.geomesa.hbase.jobs.GeoMesaHBaseInputFormat.getSplits(GeoMesaHBaseInputFormat.scala:50) at org.apache.spark.rdd.NewHadoopRDD.getPartitions(NewHadoopRDD.scala:127) at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:253) at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:251) at scala.Option.getOrElse(Option.scala:121) at org.apache.spark.rdd.RDD.partitions(RDD.scala:251) at org.apache.spark.rdd.MapPartitionsRDD.getPartitions(MapPartitionsRDD.scala:35) at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:253) at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:251) at scala.Option.getOrElse(Option.scala:121) at org.apache.spark.rdd.RDD.partitions(RDD.scala:251) at org.locationtech.geomesa.spark.SpatialRDD.getPartitions(GeoMesaSpark.scala:68) at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:253) at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:251) at scala.Option.getOrElse(Option.scala:121) at org.apache.spark.rdd.RDD.partitions(RDD.scala:251) at org.apache.spark.rdd.MapPartitionsRDD.getPartitions(MapPartitionsRDD.scala:35) at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:253) at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:251) at scala.Option.getOrElse(Option.scala:121) at org.apache.spark.rdd.RDD.partitions(RDD.scala:251) at org.apache.spark.rdd.MapPartitionsRDD.getPartitions(MapPartitionsRDD.scala:35) at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:253) at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:251) at scala.Option.getOrElse(Option.scala:121) at org.apache.spark.rdd.RDD.partitions(RDD.scala:251) at org.apache.spark.rdd.MapPartitionsRDD.getPartitions(MapPartitionsRDD.scala:35) at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:253) at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:251) at scala.Option.getOrElse(Option.scala:121) at org.apache.spark.rdd.RDD.partitions(RDD.scala:251) at org.apache.spark.rdd.MapPartitionsRDD.getPartitions(MapPartitionsRDD.scala:35) at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:253) at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:251) at scala.Option.getOrElse(Option.scala:121) at org.apache.spark.rdd.RDD.partitions(RDD.scala:251) at org.apache.spark.ShuffleDependency.<init>(Dependency.scala:91) at org.apache.spark.sql.execution.exchange.ShuffleExchangeExec$.prepareShuffleDependency(ShuffleExchangeExec.scala:319) at org.apache.spark.sql.execution.exchange.ShuffleExchangeExec.prepareShuffleDependency(ShuffleExchangeExec.scala:91) at org.apache.spark.sql.execution.exchange.ShuffleExchangeExec$$anonfun$doExecute$1.apply(ShuffleExchangeExec.scala:128) at org.apache.spark.sql.execution.exchange.ShuffleExchangeExec$$anonfun$doExecute$1.apply(ShuffleExchangeExec.scala:119) at org.apache.spark.sql.catalyst.errors.package$.attachTree(package.scala:52) ... 35 more Caused by: org.apache.hadoop.hbase.DoNotRetryIOException: java.lang.reflect.InvocationTargetException at org.apache.hadoop.hbase.protobuf.ProtobufUtil.toFilter(ProtobufUtil.java:1680) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.toScan(ProtobufUtil.java:1164) at org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil.convertStringToScan(TableMapReduceUtil.java:616) at org.apache.hadoop.hbase.mapreduce.MultiTableInputFormat.setConf(MultiTableInputFormat.java:99) ... 78 more Caused by: java.lang.reflect.InvocationTargetException at sun.reflect.GeneratedMethodAccessor121.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.toFilter(ProtobufUtil.java:1676) ... 81 more Caused by: org.apache.hadoop.hbase.exceptions.DeserializationException: org.apache.hadoop.hbase.DoNotRetryIOException: java.lang.reflect.InvocationTargetException at org.apache.hadoop.hbase.filter.FilterList.parseFrom(FilterList.java:237) ... 85 more Caused by: org.apache.hadoop.hbase.DoNotRetryIOException: java.lang.reflect.InvocationTargetException at org.apache.hadoop.hbase.protobuf.ProtobufUtil.toFilter(ProtobufUtil.java:1680) at org.apache.hadoop.hbase.filter.FilterList.parseFrom(FilterList.java:234) ... 85 more Caused by: java.lang.reflect.InvocationTargetException at sun.reflect.GeneratedMethodAccessor124.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.toFilter(ProtobufUtil.java:1676) ... 86 more Caused by: java.util.concurrent.CompletionException: org.apache.hadoop.hbase.exceptions.DeserializationException: No filter or transform defined at com.github.benmanes.caffeine.cache.BoundedLocalCache$BoundedLocalLoadingCache.lambda$new$0(BoundedLocalCache.java:3315) at com.github.benmanes.caffeine.cache.BoundedLocalCache.lambda$doComputeIfAbsent$14(BoundedLocalCache.java:2031) at java.util.concurrent.ConcurrentHashMap.compute(ConcurrentHashMap.java:1853) at com.github.benmanes.caffeine.cache.BoundedLocalCache.doComputeIfAbsent(BoundedLocalCache.java:2029) at com.github.benmanes.caffeine.cache.BoundedLocalCache.computeIfAbsent(BoundedLocalCache.java:2012) at com.github.benmanes.caffeine.cache.LocalCache.computeIfAbsent(LocalCache.java:113) at com.github.benmanes.caffeine.cache.LocalLoadingCache.get(LocalLoadingCache.java:65) at org.locationtech.geomesa.hbase.filters.CqlTransformFilter$.parseFrom(CqlTransformFilter.scala:107) at org.locationtech.geomesa.hbase.filters.CqlTransformFilter.parseFrom(CqlTransformFilter.scala) ... 90 more Caused by: org.apache.hadoop.hbase.exceptions.DeserializationException: No filter or transform defined at org.locationtech.geomesa.hbase.filters.CqlTransformFilter$.org$locationtech$geomesa$hbase$filters$CqlTransformFilter$$deserialize(CqlTransformFilter.scala:222) at org.locationtech.geomesa.hbase.filters.CqlTransformFilter$$anon$1.load(CqlTransformFilter.scala:90) at org.locationtech.geomesa.hbase.filters.CqlTransformFilter$$anon$1.load(CqlTransformFilter.scala:88) at com.github.benmanes.caffeine.cache.BoundedLocalCache$BoundedLocalLoadingCache.lambda$new$0(BoundedLocalCache.java:3308) ... 98 more During handling of the above exception, another exception occurred: Traceback (most recent call last): File "/tmp/zeppelin_pyspark-3601066725639987471.py", line 367, in <module> raise Exception(traceback.format_exc()) Exception: Traceback (most recent call last): File "/tmp/zeppelin_pyspark-3601066725639987471.py", line 360, in <module> exec(code, _zcUserQueryNameSpace) File "<stdin>", line 2, in <module> File "/usr/lib/spark/python/pyspark/sql/dataframe.py", line 455, in count return int(self._jdf.count()) File "/usr/lib/spark/python/lib/py4j-0.10.6-src.zip/py4j/java_gateway.py", line 1160, in _call_ answer, self.gateway_client, self.target_id, self.name) File "/usr/lib/spark/python/pyspark/sql/utils.py", line 63, in deco return f(*a, **kw) File "/usr/lib/spark/python/lib/py4j-0.10.6-src.zip/py4j/protocol.py", line 320, in get_return_value format(target_id, ".", name), value) py4j.protocol.Py4JJavaError: An error occurred while calling o264.count. : org.apache.spark.sql.catalyst.errors.package$TreeNodeException: execute, tree: Exchange SinglePartition +- *(1) HashAggregate(keys=[], functions=partial_count(1), output=count#1084L) +- *(1) Scan GeoMesaRelation(org.apache.spark.sql.SQLContext@634c3184,SimpleFeatureTypeImpl mover_points identified extends Feature(data_src:data_src,distance_km:distance_km,docked_port_id:docked_port_id,duration_sec:duration_sec,geom:geom,heading_rad:heading_rad,imsi:imsi,init_time:init_time,meta:meta,next_port_id:next_port_id,next_port_arrival:next_port_arrival,prev_point:prev_point,prev_port_departure:prev_port_departure,prev_port_id:prev_port_id,prev_time:prev_time,segment_id:segment_id,speed_kph:speed_kph),StructType(StructField(_fid_,StringType,false), StructField(data_src,StringType,true), StructField(distance_km,DoubleType,true), StructField(docked_port_id,StringType,true), StructField(duration_sec,IntegerType,true), StructField(geom,org.apache.spark.sql.jts.PointUDT@6a7d3f2b,true), StructField(heading_rad,DoubleType,true), StructField(imsi,StringType,true), StructField(init_time,TimestampType,true), StructField(meta,StringType,true), StructField(next_port_id,StringType,true), StructField(next_port_arrival,TimestampType,true), StructField(prev_point,org.apache.spark.sql.jts.PointUDT@6a7d3f2b,true), StructField(prev_port_departure,TimestampType,true), StructField(prev_port_id,StringType,true), StructField(prev_time,TimestampType,true), StructField(segment_id,StringType,true), StructField(speed_kph,DoubleType,true)),Map(geomesa.feature -> mover_points, hbase.catalog -> geomesa.mover),Filter.INCLUDE,None,null,null,null,null) [] PushedFilters: [], ReadSchema: struct<> at org.apache.spark.sql.catalyst.errors.package$.attachTree(package.scala:56) at org.apache.spark.sql.execution.exchange.ShuffleExchangeExec.doExecute(ShuffleExchangeExec.scala:119) at org.apache.spark.sql.execution.SparkPlan$$anonfun$execute$1.apply(SparkPlan.scala:131) at org.apache.spark.sql.execution.SparkPlan$$anonfun$execute$1.apply(SparkPlan.scala:127) at org.apache.spark.sql.execution.SparkPlan$$anonfun$executeQuery$1.apply(SparkPlan.scala:155) at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151) at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:152) at org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:127) at org.apache.spark.sql.execution.InputAdapter.inputRDDs(WholeStageCodegenExec.scala:371) at org.apache.spark.sql.execution.aggregate.HashAggregateExec.inputRDDs(HashAggregateExec.scala:150) at org.apache.spark.sql.execution.WholeStageCodegenExec.doExecute(WholeStageCodegenExec.scala:605) at org.apache.spark.sql.execution.SparkPlan$$anonfun$execute$1.apply(SparkPlan.scala:131) at org.apache.spark.sql.execution.SparkPlan$$anonfun$execute$1.apply(SparkPlan.scala:127) at org.apache.spark.sql.execution.SparkPlan$$anonfun$executeQuery$1.apply(SparkPlan.scala:155) at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151) at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:152) at org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:127) at org.apache.spark.sql.execution.SparkPlan.getByteArrayRdd(SparkPlan.scala:247) at org.apache.spark.sql.execution.SparkPlan.executeCollect(SparkPlan.scala:294) at org.apache.spark.sql.Dataset$$anonfun$count$1.apply(Dataset.scala:2770) at org.apache.spark.sql.Dataset$$anonfun$count$1.apply(Dataset.scala:2769) at org.apache.spark.sql.Dataset$$anonfun$52.apply(Dataset.scala:3253) at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:77) at org.apache.spark.sql.Dataset.withAction(Dataset.scala:3252) at org.apache.spark.sql.Dataset.count(Dataset.scala:2769) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) at py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) at py4j.Gateway.invoke(Gateway.java:282) at py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) at py4j.commands.CallCommand.execute(CallCommand.java:79) at py4j.GatewayConnection.run(GatewayConnection.java:214) at java.lang.Thread.run(Thread.java:748) Caused by: java.lang.RuntimeException: Failed to convert Scan : CgYKAWQSAWQSQAoac2Nhbi5hdHRyaWJ1dGVzLnRhYmxlLm5hbWUSImdlb21lc2EubW92ZXJfbW92 ZXJfNWZwb2ludHNfejNfdjIqhwUKKW9yZy5hcGFjaGUuaGFkb29wLmhiYXNlLmZpbHRlci5GaWx0 ZXJMaXN0EtkECAESQAoyb3JnLmFwYWNoZS5oYWRvb3AuaGJhc2UuZmlsdGVyLk11bHRpUm93UmFu Z2VGaWx0ZXISCgoICgAQARoAIAASkgQKOW9yZy5sb2NhdGlvbnRlY2guZ2VvbWVzYS5oYmFzZS5m aWx0ZXJzLkNxbFRyYW5zZm9ybUZpbHRlchLUAwAAAcRkYXRhX3NyYzpTdHJpbmcsZGlzdGFuY2Vf a206RG91YmxlLGRvY2tlZF9wb3J0X2lkOlN0cmluZyxkdXJhdGlvbl9zZWM6SW50ZWdlciwqZ2Vv bTpQb2ludDpzcmlkPTQzMjYsaGVhZGluZ19yYWQ6RG91YmxlLGltc2k6U3RyaW5nOmluZGV4PXRy dWUsaW5pdF90aW1lOkRhdGUsbWV0YTpTdHJpbmcsbmV4dF9wb3J0X2lkOlN0cmluZyxuZXh0X3Bv cnRfYXJyaXZhbDpEYXRlLHByZXZfcG9pbnQ6UG9pbnQ6c3JpZD00MzI2LHByZXZfcG9ydF9kZXBh cnR1cmU6RGF0ZSxwcmV2X3BvcnRfaWQ6U3RyaW5nLHByZXZfdGltZTpEYXRlLHNlZ21lbnRfaWQ6 U3RyaW5nOmluZGV4PXRydWUsc3BlZWRfa3BoOkRvdWJsZTtnZW9tZXNhLmluZGV4LmR0Zz0naW5p dF90aW1lJyxnZW9tZXNhLnRhYmxlLnNoYXJpbmc9J2ZhbHNlJyxnZW9tZXNhLmluZGljZXM9J3oz OjI6Myx6MjoyOjMsaWQ6MTozLGF0dHI6NTozJwAAAAAAAAAAAAAAADgBQAGIAegH to string at org.apache.hadoop.hbase.mapreduce.MultiTableInputFormat.setConf(MultiTableInputFormat.java:101) at org.locationtech.geomesa.hbase.jobs.GeoMesaHBaseInputFormat.init(GeoMesaHBaseInputFormat.scala:39) at org.locationtech.geomesa.hbase.jobs.GeoMesaHBaseInputFormat.getSplits(GeoMesaHBaseInputFormat.scala:50) at org.apache.spark.rdd.NewHadoopRDD.getPartitions(NewHadoopRDD.scala:127) at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:253) at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:251) at scala.Option.getOrElse(Option.scala:121) at org.apache.spark.rdd.RDD.partitions(RDD.scala:251) at org.apache.spark.rdd.MapPartitionsRDD.getPartitions(MapPartitionsRDD.scala:35) at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:253) at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:251) at scala.Option.getOrElse(Option.scala:121) at org.apache.spark.rdd.RDD.partitions(RDD.scala:251) at org.locationtech.geomesa.spark.SpatialRDD.getPartitions(GeoMesaSpark.scala:68) at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:253) at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:251) at scala.Option.getOrElse(Option.scala:121) at org.apache.spark.rdd.RDD.partitions(RDD.scala:251) at org.apache.spark.rdd.MapPartitionsRDD.getPartitions(MapPartitionsRDD.scala:35) at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:253) at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:251) at scala.Option.getOrElse(Option.scala:121) at org.apache.spark.rdd.RDD.partitions(RDD.scala:251) at org.apache.spark.rdd.MapPartitionsRDD.getPartitions(MapPartitionsRDD.scala:35) at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:253) at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:251) at scala.Option.getOrElse(Option.scala:121) at org.apache.spark.rdd.RDD.partitions(RDD.scala:251) at org.apache.spark.rdd.MapPartitionsRDD.getPartitions(MapPartitionsRDD.scala:35) at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:253) at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:251) at scala.Option.getOrElse(Option.scala:121) at org.apache.spark.rdd.RDD.partitions(RDD.scala:251) at org.apache.spark.rdd.MapPartitionsRDD.getPartitions(MapPartitionsRDD.scala:35) at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:253) at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:251) at scala.Option.getOrElse(Option.scala:121) at org.apache.spark.rdd.RDD.partitions(RDD.scala:251) at org.apache.spark.ShuffleDependency.<init>(Dependency.scala:91) at org.apache.spark.sql.execution.exchange.ShuffleExchangeExec$.prepareShuffleDependency(ShuffleExchangeExec.scala:319) at org.apache.spark.sql.execution.exchange.ShuffleExchangeExec.prepareShuffleDependency(ShuffleExchangeExec.scala:91) at org.apache.spark.sql.execution.exchange.ShuffleExchangeExec$$anonfun$doExecute$1.apply(ShuffleExchangeExec.scala:128) at org.apache.spark.sql.execution.exchange.ShuffleExchangeExec$$anonfun$doExecute$1.apply(ShuffleExchangeExec.scala:119) at org.apache.spark.sql.catalyst.errors.package$.attachTree(package.scala:52) ... 35 more Caused by: org.apache.hadoop.hbase.DoNotRetryIOException: java.lang.reflect.InvocationTargetException at org.apache.hadoop.hbase.protobuf.ProtobufUtil.toFilter(ProtobufUtil.java:1680) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.toScan(ProtobufUtil.java:1164) at org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil.convertStringToScan(TableMapReduceUtil.java:616) at org.apache.hadoop.hbase.mapreduce.MultiTableInputFormat.setConf(MultiTableInputFormat.java:99) ... 78 more Caused by: java.lang.reflect.InvocationTargetException at sun.reflect.GeneratedMethodAccessor121.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.toFilter(ProtobufUtil.java:1676) ... 81 more Caused by: org.apache.hadoop.hbase.exceptions.DeserializationException: org.apache.hadoop.hbase.DoNotRetryIOException: java.lang.reflect.InvocationTargetException at org.apache.hadoop.hbase.filter.FilterList.parseFrom(FilterList.java:237) ... 85 more Caused by: org.apache.hadoop.hbase.DoNotRetryIOException: java.lang.reflect.InvocationTargetException at org.apache.hadoop.hbase.protobuf.ProtobufUtil.toFilter(ProtobufUtil.java:1680) at org.apache.hadoop.hbase.filter.FilterList.parseFrom(FilterList.java:234) ... 85 more Caused by: java.lang.reflect.InvocationTargetException at sun.reflect.GeneratedMethodAccessor124.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.toFilter(ProtobufUtil.java:1676) ... 86 more Caused by: java.util.concurrent.CompletionException: org.apache.hadoop.hbase.exceptions.DeserializationException: No filter or transform defined at com.github.benmanes.caffeine.cache.BoundedLocalCache$BoundedLocalLoadingCache.lambda$new$0(BoundedLocalCache.java:3315) at com.github.benmanes.caffeine.cache.BoundedLocalCache.lambda$doComputeIfAbsent$14(BoundedLocalCache.java:2031) at java.util.concurrent.ConcurrentHashMap.compute(ConcurrentHashMap.java:1853) at com.github.benmanes.caffeine.cache.BoundedLocalCache.doComputeIfAbsent(BoundedLocalCache.java:2029) at com.github.benmanes.caffeine.cache.BoundedLocalCache.computeIfAbsent(BoundedLocalCache.java:2012) at com.github.benmanes.caffeine.cache.LocalCache.computeIfAbsent(LocalCache.java:113) at com.github.benmanes.caffeine.cache.LocalLoadingCache.get(LocalLoadingCache.java:65) at org.locationtech.geomesa.hbase.filters.CqlTransformFilter$.parseFrom(CqlTransformFilter.scala:107) at org.locationtech.geomesa.hbase.filters.CqlTransformFilter.parseFrom(CqlTransformFilter.scala) ... 90 more Caused by: org.apache.hadoop.hbase.exceptions.DeserializationException: No filter or transform defined at org.locationtech.geomesa.hbase.filters.CqlTransformFilter$.org$locationtech$geomesa$hbase$filters$CqlTransformFilter$$deserialize(CqlTransformFilter.scala:222) at org.locationtech.geomesa.hbase.filters.CqlTransformFilter$$anon$1.load(CqlTransformFilter.scala:90) at org.locationtech.geomesa.hbase.filters.CqlTransformFilter$$anon$1.load(CqlTransformFilter.scala:88) at com.github.benmanes.caffeine.cache.BoundedLocalCache$BoundedLocalLoadingCache.lambda$new$0(BoundedLocalCache.java:3308) ... 98 more ERROR Took 0 sec. Last updated by cmangold at September 19 2018, 11:42:48 AM. |