Untitled

Page 1

import org.apache.spark.sql.SparkSession

import za.co.absa.cobrix.spark.cobol.utils.SparkUtils

import spark.implicits._

val spark = SparkSession.builder().appName("Spark-Cobol Companies example").getOrCreate()

val copybook =

""" 01 COMPANY-DETAILS.

| 05 SEGMENT-ID PIC X(5).

| 05 COMPANY-ID PIC X(10).

| 05 STATIC-DETAILS.

| 10 COMPANY-NAME PIC X(15).

| 10 ADDRESS PIC X(25).

| 10 TAXPAYER.

| 15 TAXPAYER-TYPE PIC X(1).

| 15 TAXPAYER-STR PIC X(8).

| 15 TAXPAYER-NUM REDEFINES TAXPAYER-STR

| PIC 9(8) COMP.

|

| 05 CONTACTS REDEFINES STATIC-DETAILS.

| 10 PHONE-NUMBER PIC X(17).

| 10 CONTACT-PERSON PIC X(28).

|""".stripMargin

val df = spark.read.format("cobol").option("copybook_contents", copybook).option("is_record_sequence", "true")

.option("schema_retention_policy", "collapse_root") // Collapses the root group returning it's field on the top level of the schema

.option("segment_field", "SEGMENT_ID") // The SEGMENT_ID field contains IDs of segments

.option("segment_id_level0", "C") // Root segment is equals "C" [company]. All other segments are it's childern

.option("segment_id_prefix", "ID")

.load("COMP.dat")

df.show()