1WARNING: An illegal reflective access operation has occurred
2WARNING: Illegal reflective access by org.apache.spark.unsafe.Platform (file:/C:/big_data/spark-3.2.0-bin-hadoop3.2-scala2.13/jars/spark-unsafe_2.13-3.2.0.jar) to constructor java.nio.DirectByteBuffer(long,int)
3WARNING: Please consider reporting this to the maintainers of org.apache.spark.unsafe.Platform
4WARNING: Use --illegal-access=warn to enable warnings of further illegal reflective access operations
5WARNING: All illegal access operations will be denied in a future release
6Using Spark's default log4j profile: org/apache/spark/log4j-defaults.properties
7Setting default log level to "WARN".
8To adjust logging level use sc.setLogLevel(newLevel). For SparkR, use setLogLevel(newLevel).
9Welcome to
10      ____              __
11     / __/__  ___ _____/ /__
12    _\ \/ _ \/ _ `/ __/  '_/
13   /___/ .__/\_,_/_/ /_/\_\   version 3.2.0
14      /_/
15
16Using Scala version 2.13.5 (OpenJDK 64-Bit Server VM, Java 11.0.9.1)
17Type in expressions to have them evaluated.
18Type :help for more information.
1921/12/11 19:28:36 ERROR SparkContext: Error initializing SparkContext.
20java.lang.reflect.InvocationTargetException
21        at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
22        at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)
23        at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
24        at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:490)
25        at org.apache.spark.executor.Executor.addReplClassLoaderIfNeeded(Executor.scala:909)
26        at org.apache.spark.executor.Executor.<init>(Executor.scala:160)
27        at org.apache.spark.scheduler.local.LocalEndpoint.<init>(LocalSchedulerBackend.scala:64)
28        at org.apache.spark.scheduler.local.LocalSchedulerBackend.start(LocalSchedulerBackend.scala:132)
29        at org.apache.spark.scheduler.TaskSchedulerImpl.start(TaskSchedulerImpl.scala:220)
30        at org.apache.spark.SparkContext.<init>(SparkContext.scala:581)
31        at org.apache.spark.SparkContext$.getOrCreate(SparkContext.scala:2690)
32        at org.apache.spark.sql.SparkSession$Builder.$anonfun$getOrCreate$2(SparkSession.scala:949)
33        at scala.Option.getOrElse(Option.scala:201)
34        at org.apache.spark.sql.SparkSession$Builder.getOrCreate(SparkSession.scala:943)
35        at org.apache.spark.repl.Main$.createSparkSession(Main.scala:114)
36        at $line3.$read$$iw.<init>(<console>:5)
37        at $line3.$read.<init>(<console>:4)
38        at $line3.$read$.<clinit>(<console>)
39        at $line3.$eval$.$print$lzycompute(<synthetic>:6)
40        at $line3.$eval$.$print(<synthetic>:5)
41        at $line3.$eval.$print(<synthetic>)
42        at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
43        at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
44        at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
45        at java.base/java.lang.reflect.Method.invoke(Method.java:566)
46        at scala.tools.nsc.interpreter.IMain$ReadEvalPrint.call(IMain.scala:670)
47        at scala.tools.nsc.interpreter.IMain$Request.loadAndRun(IMain.scala:1006)
48        at scala.tools.nsc.interpreter.IMain.$anonfun$doInterpret$1(IMain.scala:506)
49        at scala.reflect.internal.util.ScalaClassLoader.asContext(ScalaClassLoader.scala:36)
50        at scala.reflect.internal.util.ScalaClassLoader.asContext$(ScalaClassLoader.scala:116)
51        at scala.reflect.internal.util.AbstractFileClassLoader.asContext(AbstractFileClassLoader.scala:43)
52        at scala.tools.nsc.interpreter.IMain.loadAndRunReq$1(IMain.scala:505)
53        at scala.tools.nsc.interpreter.IMain.$anonfun$doInterpret$3(IMain.scala:519)
54        at scala.tools.nsc.interpreter.IMain.doInterpret(IMain.scala:519)
55        at scala.tools.nsc.interpreter.IMain.interpret(IMain.scala:503)
56        at scala.tools.nsc.interpreter.IMain.interpret(IMain.scala:501)
57        at scala.tools.nsc.interpreter.IMain.$anonfun$quietRun$1(IMain.scala:216)
58        at scala.tools.nsc.interpreter.shell.ReplReporterImpl.withoutPrintingResults(Reporter.scala:64)
59        at scala.tools.nsc.interpreter.IMain.quietRun(IMain.scala:216)
60        at scala.tools.nsc.interpreter.shell.ILoop.$anonfun$interpretPreamble$1(ILoop.scala:924)
61        at scala.collection.immutable.List.foreach(List.scala:333)
62        at scala.tools.nsc.interpreter.shell.ILoop.interpretPreamble(ILoop.scala:924)
63        at scala.tools.nsc.interpreter.shell.ILoop.$anonfun$run$3(ILoop.scala:963)
64        at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.scala:18)
65        at scala.tools.nsc.interpreter.shell.ILoop.echoOff(ILoop.scala:90)
66        at scala.tools.nsc.interpreter.shell.ILoop.$anonfun$run$2(ILoop.scala:963)
67        at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.scala:18)
68        at scala.tools.nsc.interpreter.IMain.withSuppressedSettings(IMain.scala:1406)
69        at scala.tools.nsc.interpreter.shell.ILoop.$anonfun$run$1(ILoop.scala:954)
70        at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.scala:18)
71        at scala.tools.nsc.interpreter.shell.ReplReporterImpl.withoutPrintingResults(Reporter.scala:64)
72        at scala.tools.nsc.interpreter.shell.ILoop.run(ILoop.scala:954)
73        at org.apache.spark.repl.Main$.doMain(Main.scala:84)
74        at org.apache.spark.repl.Main$.main(Main.scala:59)
75        at org.apache.spark.repl.Main.main(Main.scala)
76        at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
77        at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
78        at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
79        at java.base/java.lang.reflect.Method.invoke(Method.java:566)
80        at org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
81        at org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:955)
82        at org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)
83        at org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)
84        at org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)
85        at org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1043)
86        at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1052)
87        at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
88Caused by: java.net.URISyntaxException: Illegal character in path at index 42: spark://DESKTOP-JO73CF4.mshome.net:2103/C:\classes
89        at java.base/java.net.URI$Parser.fail(URI.java:2913)
90        at java.base/java.net.URI$Parser.checkChars(URI.java:3084)
91        at java.base/java.net.URI$Parser.parseHierarchical(URI.java:3166)
92        at java.base/java.net.URI$Parser.parse(URI.java:3114)
93        at java.base/java.net.URI.<init>(URI.java:600)
94        at org.apache.spark.repl.ExecutorClassLoader.<init>(ExecutorClassLoader.scala:57)
95        ... 67 more
9621/12/11 19:28:36 ERROR Utils: Uncaught exception in thread main
97java.lang.NullPointerException
98        at org.apache.spark.scheduler.local.LocalSchedulerBackend.org$apache$spark$scheduler$local$LocalSchedulerBackend$$stop(LocalSchedulerBackend.scala:173)
99        at org.apache.spark.scheduler.local.LocalSchedulerBackend.stop(LocalSchedulerBackend.scala:144)
100        at org.apache.spark.scheduler.TaskSchedulerImpl.stop(TaskSchedulerImpl.scala:927)
101        at org.apache.spark.scheduler.DAGScheduler.stop(DAGScheduler.scala:2516)
102        at org.apache.spark.SparkContext.$anonfun$stop$12(SparkContext.scala:2086)
103        at org.apache.spark.util.Utils$.tryLogNonFatalError(Utils.scala:1442)
104        at org.apache.spark.SparkContext.stop(SparkContext.scala:2086)
105        at org.apache.spark.SparkContext.<init>(SparkContext.scala:677)
106        at org.apache.spark.SparkContext$.getOrCreate(SparkContext.scala:2690)
107        at org.apache.spark.sql.SparkSession$Builder.$anonfun$getOrCreate$2(SparkSession.scala:949)
108        at scala.Option.getOrElse(Option.scala:201)
109        at org.apache.spark.sql.SparkSession$Builder.getOrCreate(SparkSession.scala:943)
110        at org.apache.spark.repl.Main$.createSparkSession(Main.scala:114)
111        at $line3.$read$$iw.<init>(<console>:5)
112        at $line3.$read.<init>(<console>:4)
113        at $line3.$read$.<clinit>(<console>)
114        at $line3.$eval$.$print$lzycompute(<synthetic>:6)
115        at $line3.$eval$.$print(<synthetic>:5)
116        at $line3.$eval.$print(<synthetic>)
117        at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
118        at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
119        at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
120        at java.base/java.lang.reflect.Method.invoke(Method.java:566)
121        at scala.tools.nsc.interpreter.IMain$ReadEvalPrint.call(IMain.scala:670)
122        at scala.tools.nsc.interpreter.IMain$Request.loadAndRun(IMain.scala:1006)
123        at scala.tools.nsc.interpreter.IMain.$anonfun$doInterpret$1(IMain.scala:506)
124        at scala.reflect.internal.util.ScalaClassLoader.asContext(ScalaClassLoader.scala:36)
125        at scala.reflect.internal.util.ScalaClassLoader.asContext$(ScalaClassLoader.scala:116)
126        at scala.reflect.internal.util.AbstractFileClassLoader.asContext(AbstractFileClassLoader.scala:43)
127        at scala.tools.nsc.interpreter.IMain.loadAndRunReq$1(IMain.scala:505)
128        at scala.tools.nsc.interpreter.IMain.$anonfun$doInterpret$3(IMain.scala:519)
129        at scala.tools.nsc.interpreter.IMain.doInterpret(IMain.scala:519)
130        at scala.tools.nsc.interpreter.IMain.interpret(IMain.scala:503)
131        at scala.tools.nsc.interpreter.IMain.interpret(IMain.scala:501)
132        at scala.tools.nsc.interpreter.IMain.$anonfun$quietRun$1(IMain.scala:216)
133        at scala.tools.nsc.interpreter.shell.ReplReporterImpl.withoutPrintingResults(Reporter.scala:64)
134        at scala.tools.nsc.interpreter.IMain.quietRun(IMain.scala:216)
135        at scala.tools.nsc.interpreter.shell.ILoop.$anonfun$interpretPreamble$1(ILoop.scala:924)
136        at scala.collection.immutable.List.foreach(List.scala:333)
137        at scala.tools.nsc.interpreter.shell.ILoop.interpretPreamble(ILoop.scala:924)
138        at scala.tools.nsc.interpreter.shell.ILoop.$anonfun$run$3(ILoop.scala:963)
139        at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.scala:18)
140        at scala.tools.nsc.interpreter.shell.ILoop.echoOff(ILoop.scala:90)
141        at scala.tools.nsc.interpreter.shell.ILoop.$anonfun$run$2(ILoop.scala:963)
142        at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.scala:18)
143        at scala.tools.nsc.interpreter.IMain.withSuppressedSettings(IMain.scala:1406)
144        at scala.tools.nsc.interpreter.shell.ILoop.$anonfun$run$1(ILoop.scala:954)
145        at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.scala:18)
146        at scala.tools.nsc.interpreter.shell.ReplReporterImpl.withoutPrintingResults(Reporter.scala:64)
147        at scala.tools.nsc.interpreter.shell.ILoop.run(ILoop.scala:954)
148        at org.apache.spark.repl.Main$.doMain(Main.scala:84)
149        at org.apache.spark.repl.Main$.main(Main.scala:59)
150        at org.apache.spark.repl.Main.main(Main.scala)
151        at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
152        at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
153        at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
154        at java.base/java.lang.reflect.Method.invoke(Method.java:566)
155        at org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
156        at org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:955)
157        at org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)
158        at org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)
159        at org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)
160        at org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1043)
161        at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1052)
162        at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
16321/12/11 19:28:36 WARN MetricsSystem: Stopping a MetricsSystem that is not running
16421/12/11 19:28:36 ERROR Main: Failed to initialize Spark session.
165java.lang.reflect.InvocationTargetException
166        at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
167        at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)
168        at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
169        at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:490)
170        at org.apache.spark.executor.Executor.addReplClassLoaderIfNeeded(Executor.scala:909)
171        at org.apache.spark.executor.Executor.<init>(Executor.scala:160)
172        at org.apache.spark.scheduler.local.LocalEndpoint.<init>(LocalSchedulerBackend.scala:64)
173        at org.apache.spark.scheduler.local.LocalSchedulerBackend.start(LocalSchedulerBackend.scala:132)
174        at org.apache.spark.scheduler.TaskSchedulerImpl.start(TaskSchedulerImpl.scala:220)
175        at org.apache.spark.SparkContext.<init>(SparkContext.scala:581)
176        at org.apache.spark.SparkContext$.getOrCreate(SparkContext.scala:2690)
177        at org.apache.spark.sql.SparkSession$Builder.$anonfun$getOrCreate$2(SparkSession.scala:949)
178        at scala.Option.getOrElse(Option.scala:201)
179        at org.apache.spark.sql.SparkSession$Builder.getOrCreate(SparkSession.scala:943)
180        at org.apache.spark.repl.Main$.createSparkSession(Main.scala:114)
181        at $line3.$read$$iw.<init>(<console>:5)
182        at $line3.$read.<init>(<console>:4)
183        at $line3.$read$.<clinit>(<console>)
184        at $line3.$eval$.$print$lzycompute(<synthetic>:6)
185        at $line3.$eval$.$print(<synthetic>:5)
186        at $line3.$eval.$print(<synthetic>)
187        at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
188        at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
189        at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
190        at java.base/java.lang.reflect.Method.invoke(Method.java:566)
191        at scala.tools.nsc.interpreter.IMain$ReadEvalPrint.call(IMain.scala:670)
192        at scala.tools.nsc.interpreter.IMain$Request.loadAndRun(IMain.scala:1006)
193        at scala.tools.nsc.interpreter.IMain.$anonfun$doInterpret$1(IMain.scala:506)
194        at scala.reflect.internal.util.ScalaClassLoader.asContext(ScalaClassLoader.scala:36)
195        at scala.reflect.internal.util.ScalaClassLoader.asContext$(ScalaClassLoader.scala:116)
196        at scala.reflect.internal.util.AbstractFileClassLoader.asContext(AbstractFileClassLoader.scala:43)
197        at scala.tools.nsc.interpreter.IMain.loadAndRunReq$1(IMain.scala:505)
198        at scala.tools.nsc.interpreter.IMain.$anonfun$doInterpret$3(IMain.scala:519)
199        at scala.tools.nsc.interpreter.IMain.doInterpret(IMain.scala:519)
200        at scala.tools.nsc.interpreter.IMain.interpret(IMain.scala:503)
201        at scala.tools.nsc.interpreter.IMain.interpret(IMain.scala:501)
202        at scala.tools.nsc.interpreter.IMain.$anonfun$quietRun$1(IMain.scala:216)
203        at scala.tools.nsc.interpreter.shell.ReplReporterImpl.withoutPrintingResults(Reporter.scala:64)
204        at scala.tools.nsc.interpreter.IMain.quietRun(IMain.scala:216)
205        at scala.tools.nsc.interpreter.shell.ILoop.$anonfun$interpretPreamble$1(ILoop.scala:924)
206        at scala.collection.immutable.List.foreach(List.scala:333)
207        at scala.tools.nsc.interpreter.shell.ILoop.interpretPreamble(ILoop.scala:924)
208        at scala.tools.nsc.interpreter.shell.ILoop.$anonfun$run$3(ILoop.scala:963)
209        at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.scala:18)
210        at scala.tools.nsc.interpreter.shell.ILoop.echoOff(ILoop.scala:90)
211        at scala.tools.nsc.interpreter.shell.ILoop.$anonfun$run$2(ILoop.scala:963)
212        at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.scala:18)
213        at scala.tools.nsc.interpreter.IMain.withSuppressedSettings(IMain.scala:1406)
214        at scala.tools.nsc.interpreter.shell.ILoop.$anonfun$run$1(ILoop.scala:954)
215        at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.scala:18)
216        at scala.tools.nsc.interpreter.shell.ReplReporterImpl.withoutPrintingResults(Reporter.scala:64)
217        at scala.tools.nsc.interpreter.shell.ILoop.run(ILoop.scala:954)
218        at org.apache.spark.repl.Main$.doMain(Main.scala:84)
219        at org.apache.spark.repl.Main$.main(Main.scala:59)
220        at org.apache.spark.repl.Main.main(Main.scala)
221        at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
222        at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
223        at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
224        at java.base/java.lang.reflect.Method.invoke(Method.java:566)
225        at org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
226        at org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:955)
227        at org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)
228        at org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)
229        at org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)
230        at org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:1043)
231        at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:1052)
232        at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
233Caused by: java.net.URISyntaxException: Illegal character in path at index 42: spark://DESKTOP-JO73CF4.mshome.net:2103/C:\classes
234        at java.base/java.net.URI$Parser.fail(URI.java:2913)
235        at java.base/java.net.URI$Parser.checkChars(URI.java:3084)
236        at java.base/java.net.URI$Parser.parseHierarchical(URI.java:3166)
237        at java.base/java.net.URI$Parser.parse(URI.java:3114)
238        at java.base/java.net.URI.<init>(URI.java:600)
239        at org.apache.spark.repl.ExecutorClassLoader.<init>(ExecutorClassLoader.scala:57)
240        ... 67 more
24121/12/11 19:28:36 ERROR Utils: Uncaught exception in thread shutdown-hook-0
242java.lang.ExceptionInInitializerError
243        at org.apache.spark.executor.Executor.stop(Executor.scala:333)
244        at org.apache.spark.executor.Executor.$anonfun$stopHookReference$1(Executor.scala:76)
245        at org.apache.spark.util.SparkShutdownHook.run(ShutdownHookManager.scala:214)
246        at org.apache.spark.util.SparkShutdownHookManager.$anonfun$runAll$2(ShutdownHookManager.scala:188)
247        at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.scala:18)
248        at org.apache.spark.util.Utils$.logUncaughtExceptions(Utils.scala:2019)
249        at org.apache.spark.util.SparkShutdownHookManager.$anonfun$runAll$1(ShutdownHookManager.scala:188)
250        at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.scala:18)
251        at scala.util.Try$.apply(Try.scala:210)
252        at org.apache.spark.util.SparkShutdownHookManager.runAll(ShutdownHookManager.scala:188)
253        at org.apache.spark.util.SparkShutdownHookManager$$anon$2.run(ShutdownHookManager.scala:178)
254        at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515)
255        at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
256        at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
257        at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
258        at java.base/java.lang.Thread.run(Thread.java:829)
259Caused by: java.lang.NullPointerException
260        at org.apache.spark.shuffle.ShuffleBlockPusher$.<clinit>(ShuffleBlockPusher.scala:465)
261        ... 16 more
26221/12/11 19:28:36 WARN ShutdownHookManager: ShutdownHook '' failed, java.util.concurrent.ExecutionException: java.lang.ExceptionInInitializerError
263java.util.concurrent.ExecutionException: java.lang.ExceptionInInitializerError
264        at java.base/java.util.concurrent.FutureTask.report(FutureTask.java:122)
265        at java.base/java.util.concurrent.FutureTask.get(FutureTask.java:205)
266        at org.apache.hadoop.util.ShutdownHookManager.executeShutdown(ShutdownHookManager.java:124)
267        at org.apache.hadoop.util.ShutdownHookManager$1.run(ShutdownHookManager.java:95)
268Caused by: java.lang.ExceptionInInitializerError
269        at org.apache.spark.executor.Executor.stop(Executor.scala:333)
270        at org.apache.spark.executor.Executor.$anonfun$stopHookReference$1(Executor.scala:76)
271        at org.apache.spark.util.SparkShutdownHook.run(ShutdownHookManager.scala:214)
272        at org.apache.spark.util.SparkShutdownHookManager.$anonfun$runAll$2(ShutdownHookManager.scala:188)
273        at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.scala:18)
274        at org.apache.spark.util.Utils$.logUncaughtExceptions(Utils.scala:2019)
275        at org.apache.spark.util.SparkShutdownHookManager.$anonfun$runAll$1(ShutdownHookManager.scala:188)
276        at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.scala:18)
277        at scala.util.Try$.apply(Try.scala:210)
278        at org.apache.spark.util.SparkShutdownHookManager.runAll(ShutdownHookManager.scala:188)
279        at org.apache.spark.util.SparkShutdownHookManager$$anon$2.run(ShutdownHookManager.scala:178)
280        at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515)
281        at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
282        at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
283        at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
284        at java.base/java.lang.Thread.run(Thread.java:829)
285Caused by: java.lang.NullPointerException
286        at org.apache.spark.shuffle.ShuffleBlockPusher$.<clinit>(ShuffleBlockPusher.scala:465)
287        ... 16 more
288

1data = [
2    (1, 'bad', 'A'),
3    (4, 'ok', None)]
4df1 = spark.createDataFrame(data, ['ID', 'Status', 'c1'])
5df1 = df1.withColumn('c2', F.lit('A'))
6df1.show()
7
8#+---+------+----+---+
9#| ID|Status|  c1| c2|
10#+---+------+----+---+
11#|  1|   bad|   A|  A|
12#|  4|    ok|null|  A|
13#+---+------+----+---+
14
15df2 = df1.filter((F.col('Status') == 'ok'))
16df2.show()
17
18#+---+------+----+---+
19#| ID|Status|  c1| c2|
20#+---+------+----+---+
21#|  4|    ok|null|  A|
22#+---+------+----+---+
23
24df3 = df2.join(df1, (df1.c1 == df2.c2), 'full')
25df3.show()
26
27#+----+------+----+----+----+------+----+----+
28#|  ID|Status|  c1|  c2|  ID|Status|  c1|  c2|
29#+----+------+----+----+----+------+----+----+
30#|   4|    ok|null|   A|null|  null|null|null|
31#|null|  null|null|null|   1|   bad|   A|   A|
32#|null|  null|null|null|   4|    ok|null|   A|
33#+----+------+----+----+----+------+----+----+
34

1data = [
2    (1, 'bad', 'A'),
3    (4, 'ok', None)]
4df1 = spark.createDataFrame(data, ['ID', 'Status', 'c1'])
5df1 = df1.withColumn('c2', F.lit('A'))
6df1.show()
7
8#+---+------+----+---+
9#| ID|Status|  c1| c2|
10#+---+------+----+---+
11#|  1|   bad|   A|  A|
12#|  4|    ok|null|  A|
13#+---+------+----+---+
14
15df2 = df1.filter((F.col('Status') == 'ok'))
16df2.show()
17
18#+---+------+----+---+
19#| ID|Status|  c1| c2|
20#+---+------+----+---+
21#|  4|    ok|null|  A|
22#+---+------+----+---+
23
24df3 = df2.join(df1, (df1.c1 == df2.c2), 'full')
25df3.show()
26
27#+----+------+----+----+----+------+----+----+
28#|  ID|Status|  c1|  c2|  ID|Status|  c1|  c2|
29#+----+------+----+----+----+------+----+----+
30#|   4|    ok|null|   A|null|  null|null|null|
31#|null|  null|null|null|   1|   bad|   A|   A|
32#|null|  null|null|null|   4|    ok|null|   A|
33#+----+------+----+----+----+------+----+----+
34data = [
35    (1, 'bad', 'A', 'A'),
36    (4, 'ok', None, 'A')]
37df4 = spark.createDataFrame(data, ['ID', 'Status', 'c1', 'c2'])
38df4.show()
39
40#+---+------+----+---+
41#| ID|Status|  c1| c2|
42#+---+------+----+---+
43#|  1|   bad|   A|  A|
44#|  4|    ok|null|  A|
45#+---+------+----+---+
46
47df5 = spark.createDataFrame(data, ['ID', 'Status', 'c1', 'c2']).filter((F.col('Status') == 'ok'))
48df5.show()
49
50#+---+------+----+---+
51#| ID|Status|  c1| c2|
52#+---+------+----+---+
53#|  4|    ok|null|  A|
54#+---+------+----+---+
55
56df6 = df5.join(df4, (df4.c1 == df5.c2), 'full')
57df6.show()
58
59#+----+------+----+----+---+------+----+---+
60#|  ID|Status|  c1|  c2| ID|Status|  c1| c2|
61#+----+------+----+----+---+------+----+---+
62#|null|  null|null|null|  4|    ok|null|  A|
63#|   4|    ok|null|   A|  1|   bad|   A|  A|
64#+----+------+----+----+---+------+----+---+
65

1data = [
2    (1, 'bad', 'A'),
3    (4, 'ok', None)]
4df1 = spark.createDataFrame(data, ['ID', 'Status', 'c1'])
5df1 = df1.withColumn('c2', F.lit('A'))
6df1.show()
7
8#+---+------+----+---+
9#| ID|Status|  c1| c2|
10#+---+------+----+---+
11#|  1|   bad|   A|  A|
12#|  4|    ok|null|  A|
13#+---+------+----+---+
14
15df2 = df1.filter((F.col('Status') == 'ok'))
16df2.show()
17
18#+---+------+----+---+
19#| ID|Status|  c1| c2|
20#+---+------+----+---+
21#|  4|    ok|null|  A|
22#+---+------+----+---+
23
24df3 = df2.join(df1, (df1.c1 == df2.c2), 'full')
25df3.show()
26
27#+----+------+----+----+----+------+----+----+
28#|  ID|Status|  c1|  c2|  ID|Status|  c1|  c2|
29#+----+------+----+----+----+------+----+----+
30#|   4|    ok|null|   A|null|  null|null|null|
31#|null|  null|null|null|   1|   bad|   A|   A|
32#|null|  null|null|null|   4|    ok|null|   A|
33#+----+------+----+----+----+------+----+----+
34data = [
35    (1, 'bad', 'A', 'A'),
36    (4, 'ok', None, 'A')]
37df4 = spark.createDataFrame(data, ['ID', 'Status', 'c1', 'c2'])
38df4.show()
39
40#+---+------+----+---+
41#| ID|Status|  c1| c2|
42#+---+------+----+---+
43#|  1|   bad|   A|  A|
44#|  4|    ok|null|  A|
45#+---+------+----+---+
46
47df5 = spark.createDataFrame(data, ['ID', 'Status', 'c1', 'c2']).filter((F.col('Status') == 'ok'))
48df5.show()
49
50#+---+------+----+---+
51#| ID|Status|  c1| c2|
52#+---+------+----+---+
53#|  4|    ok|null|  A|
54#+---+------+----+---+
55
56df6 = df5.join(df4, (df4.c1 == df5.c2), 'full')
57df6.show()
58
59#+----+------+----+----+---+------+----+---+
60#|  ID|Status|  c1|  c2| ID|Status|  c1| c2|
61#+----+------+----+----+---+------+----+---+
62#|null|  null|null|null|  4|    ok|null|  A|
63#|   4|    ok|null|   A|  1|   bad|   A|  A|
64#+----+------+----+----+---+------+----+---+
65df3.explain()
66
67== Physical Plan ==
68BroadcastNestedLoopJoin BuildRight, FullOuter, (c1#23335 = A)
69:- *(1) Project [ID#23333L, Status#23334, c1#23335, A AS c2#23339]
70:  +- *(1) Filter (isnotnull(Status#23334) AND (Status#23334 = ok))
71:     +- *(1) Scan ExistingRDD[ID#23333L,Status#23334,c1#23335]
72+- BroadcastExchange IdentityBroadcastMode, [id=#9250]
73   +- *(2) Project [ID#23379L, Status#23380, c1#23381, A AS c2#23378]
74      +- *(2) Scan ExistingRDD[ID#23379L,Status#23380,c1#23381]
75
76df6.explain()
77
78== Physical Plan ==
79SortMergeJoin [c2#23459], [c1#23433], FullOuter
80:- *(2) Sort [c2#23459 ASC NULLS FIRST], false, 0
81:  +- Exchange hashpartitioning(c2#23459, 200), ENSURE_REQUIREMENTS, [id=#9347]
82:     +- *(1) Filter (isnotnull(Status#23457) AND (Status#23457 = ok))
83:        +- *(1) Scan ExistingRDD[ID#23456L,Status#23457,c1#23458,c2#23459]
84+- *(4) Sort [c1#23433 ASC NULLS FIRST], false, 0
85   +- Exchange hashpartitioning(c1#23433, 200), ENSURE_REQUIREMENTS, [id=#9352]
86      +- *(3) Scan ExistingRDD[ID#23431L,Status#23432,c1#23433,c2#23434]
87

1search =  SearchEngine(db_file_dir = "/tmp/db")
2conn = sqlite3.connect("/tmp/db/simple_db.sqlite")
3pdf_ = pd.read_sql_query('''select  zipcode, lat, lng, 
4                        bounds_west, bounds_east, bounds_north, bounds_south from 
5                        simple_zipcode''',conn)
6brd_pdf = spark.sparkContext.broadcast(pdf_) 
7conn.close()
8
9
10@udf('string')
11def get_zip_b(lat, lng):
12    pdf = brd_pdf.value 
13    out = pdf[(np.array(pdf["bounds_north"]) >= lat) & 
14              (np.array(pdf["bounds_south"]) <= lat) & 
15              (np.array(pdf['bounds_west']) <= lng) & 
16              (np.array(pdf['bounds_east']) >= lng) ]
17    if len(out):
18        min_index = np.argmin( (np.array(out["lat"]) - lat)**2 + (np.array(out["lng"]) - lng)**2)
19        zip_ = str(out["zipcode"].iloc[min_index])
20    else:
21        zip_ = 'bad'
22    return zip_
23
24df = df.withColumn('zipcode', get_zip_b(col("latitude"),col("longitude")))
25

1search =  SearchEngine(db_file_dir = "/tmp/db")
2conn = sqlite3.connect("/tmp/db/simple_db.sqlite")
3pdf_ = pd.read_sql_query('''select  zipcode, lat, lng, 
4                        bounds_west, bounds_east, bounds_north, bounds_south from 
5                        simple_zipcode''',conn)
6brd_pdf = spark.sparkContext.broadcast(pdf_) 
7conn.close()
8
9
10@udf('string')
11def get_zip_b(lat, lng):
12    pdf = brd_pdf.value 
13    out = pdf[(np.array(pdf["bounds_north"]) >= lat) & 
14              (np.array(pdf["bounds_south"]) <= lat) & 
15              (np.array(pdf['bounds_west']) <= lng) & 
16              (np.array(pdf['bounds_east']) >= lng) ]
17    if len(out):
18        min_index = np.argmin( (np.array(out["lat"]) - lat)**2 + (np.array(out["lng"]) - lng)**2)
19        zip_ = str(out["zipcode"].iloc[min_index])
20    else:
21        zip_ = 'bad'
22    return zip_
23
24df = df.withColumn('zipcode', get_zip_b(col("latitude"),col("longitude")))
2521/08/02 06:18:19 WARN TaskSetManager: Lost task 12.0 in stage 7.0 (TID 1814, ip-10-22-17-94.pclc0.merkle.local, executor 6): org.apache.spark.api.python.PythonException: Traceback (most recent call last):
26  File "/mnt/yarn/usercache/hadoop/appcache/application_1627867699893_0001/container_1627867699893_0001_01_000009/pyspark.zip/pyspark/worker.py", line 605, in main
27    process()
28  File "/mnt/yarn/usercache/hadoop/appcache/application_1627867699893_0001/container_1627867699893_0001_01_000009/pyspark.zip/pyspark/worker.py", line 597, in process
29    serializer.dump_stream(out_iter, outfile)
30  File "/mnt/yarn/usercache/hadoop/appcache/application_1627867699893_0001/container_1627867699893_0001_01_000009/pyspark.zip/pyspark/serializers.py", line 223, in dump_stream
31    self.serializer.dump_stream(self._batched(iterator), stream)
32  File "/mnt/yarn/usercache/hadoop/appcache/application_1627867699893_0001/container_1627867699893_0001_01_000009/pyspark.zip/pyspark/serializers.py", line 141, in dump_stream
33    for obj in iterator:
34  File "/mnt/yarn/usercache/hadoop/appcache/application_1627867699893_0001/container_1627867699893_0001_01_000009/pyspark.zip/pyspark/serializers.py", line 212, in _batched
35    for item in iterator:
36  File "/mnt/yarn/usercache/hadoop/appcache/application_1627867699893_0001/container_1627867699893_0001_01_000009/pyspark.zip/pyspark/worker.py", line 450, in mapper
37    result = tuple(f(*[a[o] for o in arg_offsets]) for (arg_offsets, f) in udfs)
38  File "/mnt/yarn/usercache/hadoop/appcache/application_1627867699893_0001/container_1627867699893_0001_01_000009/pyspark.zip/pyspark/worker.py", line 450, in <genexpr>
39    result = tuple(f(*[a[o] for o in arg_offsets]) for (arg_offsets, f) in udfs)
40  File "/mnt/yarn/usercache/hadoop/appcache/application_1627867699893_0001/container_1627867699893_0001_01_000009/pyspark.zip/pyspark/worker.py", line 90, in <lambda>
41    return lambda *a: f(*a)
42  File "/mnt/yarn/usercache/hadoop/appcache/application_1627867699893_0001/container_1627867699893_0001_01_000009/pyspark.zip/pyspark/util.py", line 121, in wrapper
43    return f(*args, **kwargs)
44  File "/mnt/var/lib/hadoop/steps/s-1IBFS0SYWA19Z/Mobile_ID_process_center.py", line 102, in get_zip_b
45  File "/mnt/yarn/usercache/hadoop/appcache/application_1627867699893_0001/container_1627867699893_0001_01_000009/pyspark.zip/pyspark/broadcast.py", line 146, in value
46    self._value = self.load_from_path(self._path)
47  File "/mnt/yarn/usercache/hadoop/appcache/application_1627867699893_0001/container_1627867699893_0001_01_000009/pyspark.zip/pyspark/broadcast.py", line 123, in load_from_path
48    return self.load(f)
49  File "/mnt/yarn/usercache/hadoop/appcache/application_1627867699893_0001/container_1627867699893_0001_01_000009/pyspark.zip/pyspark/broadcast.py", line 129, in load
50    return pickle.load(file)
51AttributeError: Can't get attribute 'new_block' on <module 'pandas.core.internals.blocks' from '/mnt/miniconda/lib/python3.9/site-packages/pandas/core/internals/blocks.py'>
52

1An error occurred while calling z:org.apache.spark.api.python.PythonRDD.runJob.
2You may get a different result due to the upgrading of Spark 3.0: reading dates before 1582-10-15 or timestamps before 1900-01-01T00:00:00Z from Parquet INT96 files can be ambiguous, 
3as the files may be written by Spark 2.x or legacy versions of Hive, which uses a legacy hybrid calendar that is different from Spark 3.0+s Proleptic Gregorian calendar.
4See more details in SPARK-31404.
5You can set spark.sql.legacy.parquet.int96RebaseModeInRead to 'LEGACY' to rebase the datetime values w.r.t. the calendar difference during reading. 
6Or set spark.sql.legacy.parquet.int96RebaseModeInRead to 'CORRECTED' to read the datetime values as it is.
7

1import ch.qos.logback.classic._
2import com.typesafe.scalalogging._
3import com.fasterxml.jackson._
4import com.fasterxml.jackson.core._
5import com.fasterxml.jackson.databind.{ObjectMapper, JsonNode}
6import com.fasterxml.jackson.dataformat.xml._
7import com.fasterxml.jackson.module.scala._
8import com.fasterxml.jackson.module.scala.experimental.ScalaObjectMapper
9import java.io._
10import java.time.Instant
11import java.util.concurrent.TimeUnit
12import javax.xml.parsers._
13import okhttp3.{Headers, OkHttpClient, Request, Response, RequestBody, FormBody}
14import okhttp3.OkHttpClient.Builder._
15import org.apache.spark._
16import org.xml.sax._
17

1import ch.qos.logback.classic._
2import com.typesafe.scalalogging._
3import com.fasterxml.jackson._
4import com.fasterxml.jackson.core._
5import com.fasterxml.jackson.databind.{ObjectMapper, JsonNode}
6import com.fasterxml.jackson.dataformat.xml._
7import com.fasterxml.jackson.module.scala._
8import com.fasterxml.jackson.module.scala.experimental.ScalaObjectMapper
9import java.io._
10import java.time.Instant
11import java.util.concurrent.TimeUnit
12import javax.xml.parsers._
13import okhttp3.{Headers, OkHttpClient, Request, Response, RequestBody, FormBody}
14import okhttp3.OkHttpClient.Builder._
15import org.apache.spark._
16import org.xml.sax._
17com.squareup.okhttp:okhttp:2.7.5
18com.squareup.okhttp3:okhttp:4.9.0
19com.squareup.okhttp3:okhttp:3.14.9
20org.scala-lang.modules:scala-swing_3:3.0.0
21ch.qos.logback:logback-classic:1.2.6
22com.typesafe:scalalogging-slf4j_2.10:1.1.0
23cc.spray.json:spray-json_2.9.1:1.0.1
24com.fasterxml.jackson.module:jackson-module-scala_3:2.13.0
25javax.xml.parsers:jaxp-api:1.4.5
26org.xml.sax:2.0.1
27

1import ch.qos.logback.classic._
2import com.typesafe.scalalogging._
3import com.fasterxml.jackson._
4import com.fasterxml.jackson.core._
5import com.fasterxml.jackson.databind.{ObjectMapper, JsonNode}
6import com.fasterxml.jackson.dataformat.xml._
7import com.fasterxml.jackson.module.scala._
8import com.fasterxml.jackson.module.scala.experimental.ScalaObjectMapper
9import java.io._
10import java.time.Instant
11import java.util.concurrent.TimeUnit
12import javax.xml.parsers._
13import okhttp3.{Headers, OkHttpClient, Request, Response, RequestBody, FormBody}
14import okhttp3.OkHttpClient.Builder._
15import org.apache.spark._
16import org.xml.sax._
17com.squareup.okhttp:okhttp:2.7.5
18com.squareup.okhttp3:okhttp:4.9.0
19com.squareup.okhttp3:okhttp:3.14.9
20org.scala-lang.modules:scala-swing_3:3.0.0
21ch.qos.logback:logback-classic:1.2.6
22com.typesafe:scalalogging-slf4j_2.10:1.1.0
23cc.spray.json:spray-json_2.9.1:1.0.1
24com.fasterxml.jackson.module:jackson-module-scala_3:2.13.0
25javax.xml.parsers:jaxp-api:1.4.5
26org.xml.sax:2.0.1
27val xmlMapper: XmlMapper = new XmlMapper()
28val jsonNode: JsonNode = xmlMapper.readTree(responseBody.getBytes())
29

1    "classification": "spark-env",
2    "properties": {
3        "conda": "/home/hadoop/conda/bin"
4    }
5

1    "classification": "spark-env",
2    "properties": {
3        "conda": "/home/hadoop/conda/bin"
4    }
5[hadoop@ip-172-30-5-150 ~]$ PATH=/home/hadoop/conda/bin:$PATH
6[hadoop@ip-172-30-5-150 ~]$ conda
7usage: conda [-h] [-V] command ...
8
9conda is a tool for managing and deploying applications, environments and packages.
10

1    "classification": "spark-env",
2    "properties": {
3        "conda": "/home/hadoop/conda/bin"
4    }
5[hadoop@ip-172-30-5-150 ~]$ PATH=/home/hadoop/conda/bin:$PATH
6[hadoop@ip-172-30-5-150 ~]$ conda
7usage: conda [-h] [-V] command ...
8
9conda is a tool for managing and deploying applications, environments and packages.
10#!/usr/bin/env bash
11
12
13# Install conda
14wget https://repo.continuum.io/miniconda/Miniconda3-4.2.12-Linux-x86_64.sh -O /home/hadoop/miniconda.sh \
15    && /bin/bash ~/miniconda.sh -b -p $HOME/conda
16
17
18conda config --set always_yes yes --set changeps1 no
19conda install conda=4.2.13
20conda config -f --add channels conda-forge
21rm ~/miniconda.sh
22echo bootstrap_conda.sh completed. PATH now: $PATH
23export PYSPARK_PYTHON="/home/hadoop/conda/bin/python3.5"
24
25echo -e '\nexport PATH=$HOME/conda/bin:$PATH' >> $HOME/.bashrc && source $HOME/.bashrc
26
27
28conda create -n zoo python=3.7 # "zoo" is conda environment name, you can use any name you like.
29conda activate zoo
30sudo pip3 install tensorflow
31sudo pip3 install boto3
32sudo pip3 install botocore
33sudo pip3 install numpy
34sudo pip3 install pandas
35sudo pip3 install scipy
36sudo pip3 install s3fs
37sudo pip3 install matplotlib
38sudo pip3 install -U tqdm
39sudo pip3 install -U scikit-learn
40sudo pip3 install -U scikit-multilearn
41sudo pip3 install xlutils
42sudo pip3 install natsort
43sudo pip3 install pydot
44sudo pip3 install python-pydot
45sudo pip3 install python-pydot-ng
46sudo pip3 install pydotplus
47sudo pip3 install h5py
48sudo pip3 install graphviz
49sudo pip3 install recmetrics
50sudo pip3 install openpyxl
51sudo pip3 install xlrd
52sudo pip3 install xlwt
53sudo pip3 install tensorflow.io
54sudo pip3 install Cython
55sudo pip3 install ray
56sudo pip3 install zoo
57sudo pip3 install analytics-zoo
58sudo pip3 install analytics-zoo[ray]
59#sudo /usr/bin/pip-3.6 install -U imbalanced-learn
60
61
62

1docker compose \
2  --file docker-compose.yml \
3  --file backend/docker-compose.yml \
4  --file docker-compose.override.yml up
5

1docker compose \
2  --file docker-compose.yml \
3  --file backend/docker-compose.yml \
4  --file docker-compose.override.yml up
5/docker-compose.yml
6/docker-compose.override.yml
7

1docker compose \
2  --file docker-compose.yml \
3  --file backend/docker-compose.yml \
4  --file docker-compose.override.yml up
5/docker-compose.yml
6/docker-compose.override.yml
7/docker-compose.yml
8/docker-compose.override.yml
9/backend/ # pulled in as a Git submodule
10/backend/docker-compose.yml
11/backend/docker-compose.override.yml
12

1prefix/company=abcd/service=xyz/date=2021-01-01/file_01.json
2prefix/company=abcd/service=xyz/date=2021-01-01/file_02.json
3prefix/company=abcd/service=xyz/date=2021-01-01/file_03.json
4

1prefix/company=abcd/service=xyz/date=2021-01-01/file_01.json
2prefix/company=abcd/service=xyz/date=2021-01-01/file_02.json
3prefix/company=abcd/service=xyz/date=2021-01-01/file_03.json
4self.spark \
5    .read \
6    .option("basePath", 'prefix') \
7    .schema(self.schema) \
8    .json(['company=abcd/service=xyz/date=2021-01-01/'])
9

1prefix/company=abcd/service=xyz/date=2021-01-01/file_01.json
2prefix/company=abcd/service=xyz/date=2021-01-01/file_02.json
3prefix/company=abcd/service=xyz/date=2021-01-01/file_03.json
4self.spark \
5    .read \
6    .option("basePath", 'prefix') \
7    .schema(self.schema) \
8    .json(['company=abcd/service=xyz/date=2021-01-01/'])
9{"id": "foo", "color": "blue", "date": "2021-12-12"}
10

1prefix/company=abcd/service=xyz/date=2021-01-01/file_01.json
2prefix/company=abcd/service=xyz/date=2021-01-01/file_02.json
3prefix/company=abcd/service=xyz/date=2021-01-01/file_03.json
4self.spark \
5    .read \
6    .option("basePath", 'prefix') \
7    .schema(self.schema) \
8    .json(['company=abcd/service=xyz/date=2021-01-01/'])
9{"id": "foo", "color": "blue", "date": "2021-12-12"}
10| id  | color | file_date  | company | service | date       |
11-------------------------------------------------------------
12| foo | blue  | 2021-12-12 | abcd    | xyz     | 2021-01-01 |
13| bar | red   | 2021-10-10 | abcd    | xyz     | 2021-01-01 |
14| baz | green | 2021-08-08 | abcd    | xyz     | 2021-01-01 |
15

1FROM node:16-buster-slim
2RUN apt-get update
3
4WORKDIR /app
5COPY package.json /home
6RUN npm install --prefix /home
7

1FROM node:16-buster-slim
2RUN apt-get update
3
4WORKDIR /app
5COPY package.json /home
6RUN npm install --prefix /home
7{
8  "name": "test",
9  "version": "0.1.0",
10  "private": false,
11  "scripts": {
12    "dev": "vue-cli-service serve --mode development --host 0.0.0.0",
13    "serve": "vue-cli-service serve --mode production --host 0.0.0.0",
14    "build": "vue-cli-service build",
15    "jest": "vue-cli-service test:unit --watchAll",
16    "lint": "vue-cli-service lint"
17  },
18  "dependencies": {
19    "axios": "^0.21.4",
20    "core-js": "^3.6.5",
21    "dateformat": "^5.0.2",
22    "element-plus": "^1.1.0-beta.9",
23    "element-ui": "^2.15.6",
24    "lib-flexible": "^0.3.2",
25    "ol": "^6.6.1",
26    "spark-md5": "^3.0.2",
27    "vue": "^3.0.0",
28    "vue-router": "^4.0.0-0",
29    "vuelayers": "^0.11.36",
30    "vuex": "^4.0.0-0"
31  },
32  "devDependencies": {
33    "@vue/cli-plugin-babel": "~4.5.0",
34    "@vue/cli-plugin-eslint": "~4.5.0",
35    "@vue/cli-plugin-router": "~4.5.0",
36    "@vue/cli-plugin-unit-jest": "~4.5.0",
37    "@vue/cli-plugin-vuex": "~4.5.0",
38    "@vue/cli-service": "~4.5.0",
39    "@vue/compiler-sfc": "^3.0.0",
40    "@vue/eslint-config-standard": "^5.1.2",
41    "@vue/test-utils": "^2.0.0-0",
42    "babel-eslint": "^10.1.0",
43    "babel-plugin-component": "^1.1.1",
44    "eslint": "^6.7.2",
45    "eslint-plugin-import": "^2.20.2",
46    "eslint-plugin-node": "^11.1.0",
47    "eslint-plugin-promise": "^4.2.1",
48    "eslint-plugin-standard": "^4.0.0",
49    "eslint-plugin-vue": "^7.0.0",
50    "fs-extra": "^10.0.0",
51    "lint-staged": "^9.5.0",
52    "mockjs": "^1.1.0",
53    "node-sass": "^4.14.1",
54    "px2rem-loader": "^0.1.9",
55    "sass-loader": "^8.0.2",
56    "typescript": "~3.9.3",
57    "vue-jest": "^5.0.0-0"
58  },
59  "gitHooks": {
60    "pre-commit": "lint-staged"
61  },
62  "lint-staged": {
63    "*.{js,jsx,vue}": [
64      "vue-cli-service lint",
65      "git add"
66    ]
67  }
68}
69
70

1FROM node:16-buster-slim
2RUN apt-get update
3
4WORKDIR /app
5COPY package.json /home
6RUN npm install --prefix /home
7{
8  "name": "test",
9  "version": "0.1.0",
10  "private": false,
11  "scripts": {
12    "dev": "vue-cli-service serve --mode development --host 0.0.0.0",
13    "serve": "vue-cli-service serve --mode production --host 0.0.0.0",
14    "build": "vue-cli-service build",
15    "jest": "vue-cli-service test:unit --watchAll",
16    "lint": "vue-cli-service lint"
17  },
18  "dependencies": {
19    "axios": "^0.21.4",
20    "core-js": "^3.6.5",
21    "dateformat": "^5.0.2",
22    "element-plus": "^1.1.0-beta.9",
23    "element-ui": "^2.15.6",
24    "lib-flexible": "^0.3.2",
25    "ol": "^6.6.1",
26    "spark-md5": "^3.0.2",
27    "vue": "^3.0.0",
28    "vue-router": "^4.0.0-0",
29    "vuelayers": "^0.11.36",
30    "vuex": "^4.0.0-0"
31  },
32  "devDependencies": {
33    "@vue/cli-plugin-babel": "~4.5.0",
34    "@vue/cli-plugin-eslint": "~4.5.0",
35    "@vue/cli-plugin-router": "~4.5.0",
36    "@vue/cli-plugin-unit-jest": "~4.5.0",
37    "@vue/cli-plugin-vuex": "~4.5.0",
38    "@vue/cli-service": "~4.5.0",
39    "@vue/compiler-sfc": "^3.0.0",
40    "@vue/eslint-config-standard": "^5.1.2",
41    "@vue/test-utils": "^2.0.0-0",
42    "babel-eslint": "^10.1.0",
43    "babel-plugin-component": "^1.1.1",
44    "eslint": "^6.7.2",
45    "eslint-plugin-import": "^2.20.2",
46    "eslint-plugin-node": "^11.1.0",
47    "eslint-plugin-promise": "^4.2.1",
48    "eslint-plugin-standard": "^4.0.0",
49    "eslint-plugin-vue": "^7.0.0",
50    "fs-extra": "^10.0.0",
51    "lint-staged": "^9.5.0",
52    "mockjs": "^1.1.0",
53    "node-sass": "^4.14.1",
54    "px2rem-loader": "^0.1.9",
55    "sass-loader": "^8.0.2",
56    "typescript": "~3.9.3",
57    "vue-jest": "^5.0.0-0"
58  },
59  "gitHooks": {
60    "pre-commit": "lint-staged"
61  },
62  "lint-staged": {
63    "*.{js,jsx,vue}": [
64      "vue-cli-service lint",
65      "git add"
66    ]
67  }
68}
69
70                                                                                                                                                                                                                                                                                                        npm ERR! A complete log of this run can be found in:                  
71