Convert HallOfBeornDump to CSV for ALEP pipeline

cmd/beornextract/BUILD

@@ -14,7 +14,8 @@ go_library(
     importpath = "github.com/squk/lotr/cmd/beornextract",
     deps = [
         "//cmd/beornextract/types",
-        "@com_github_davecgh_go_spew//spew",
+        "@com_github_grokify_html_strip_tags_go//:html-strip-tags-go",
+        "@com_github_jessevdk_go_flags//:go_default_library",
         "@io_bazel_rules_go//go/tools/bazel:go_default_library",
     ],
 )

cmd/beornextract/main.go

@@ -1,19 +1,37 @@
 package main
 
 import (
+	"encoding/csv"
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
 	"log"
 	"os"
 	"path/filepath"
+	"regexp"
+	"strconv"
+	"strings"
 
 	"github.com/bazelbuild/rules_go/go/tools/bazel"
-	"github.com/davecgh/go-spew/spew"
+	strip "github.com/grokify/html-strip-tags-go"
+	"github.com/jessevdk/go-flags"
 	"github.com/squk/lotr/cmd/beornextract/types"
 )
 
+type Options struct {
+	RawConversion bool `short:"r" long:"raw" description:"Enable to keep the original text from HallOfBeorn dump. Enable to prep for ALEP pipeline."`
+}
+
+var opts = Options{
+	RawConversion: false,
+}
+
 func main() {
+	_, err := flags.ParseArgs(&opts, os.Args)
+	if err != nil {
+		panic(err)
+	}
+
 	fmt.Println("LOTR CARD PARSE")
 	f, err := bazel.Runfile(".")
 	if err != nil {
@@ -33,18 +51,86 @@ func main() {
 
 	// Open our jsonFile
 	// jsonFile, err := os.Open("cmd/beornextract/data/Bot.Cards.json")
-	jsonFile, err := bazel.Runfile("cmd/beornextract/data/Bot.Cards.json")
+	jsonFile, err := os.Open("cmd/beornextract/data/Export.Cards.json")
 	// if we os.Open returns an error then handle it
 	if err != nil {
 		fmt.Println(err)
 		return
 	}
-	fmt.Println("Successfully Opened users.json")
 	// defer the closing of our jsonFile so that we can parse it later on
 	defer jsonFile.Close()
 
 	byteValue, _ := ioutil.ReadAll(jsonFile)
-	c := []types.HallOfBeornCard{}
-	json.Unmarshal(byteValue, &c)
-	spew.Dump(c)
+	cards := []types.HallOfBeornCard{}
+	json.Unmarshal(byteValue, &cards)
+
+	// Open a file for writing
+	csvFile, err := os.Create("/Users/christian/Downloads/lotr-lcg-set-generator.csv")
+	defer csvFile.Close()
+	if err != nil {
+		// Handle error
+	}
+
+	// Create a writer
+	w := csv.NewWriter(csvFile)
+
+	// Write some rows
+	for _, card := range cards {
+		if card.EncounterSet != "" {
+			continue // skip non=player cards
+		}
+
+		w.Write(
+			[]string{
+				card.Octgnid,
+				"", // hidden
+				"", // hidden
+				card.EncounterSet,
+				strconv.Itoa(card.Position),
+				strconv.Itoa(card.Quantity),
+				card.Name,
+				fmt.Sprintf(
+					"%t",
+					card.IsUnique,
+				),
+				card.TypeName,
+				card.SphereName,
+				card.Traits,
+				findKeywords(card.Text),
+				card.Cost,
+				card.EngagementCost,
+				strconv.Itoa(card.Threat),
+				strconv.Itoa(card.Willpower),
+				strconv.Itoa(card.Attack),
+				strconv.Itoa(card.Defense),
+				strconv.Itoa(card.Health),
+				card.QuestPoints,
+				strconv.Itoa(card.VictoryPoints),
+				"", // Special Icon
+				transformText(card.Name, card.Text),
+				card.Flavor,
+			},
+		)
+	}
+
+	// Close the writer
+	w.Flush()
+	// spew.Dump(c)
+}
+
+func transformText(name, text string) string {
+	if opts.RawConversion {
+		return text
+	}
+
+	out := strings.ReplaceAll(text, name, "[name]") // insert name tag
+	out = strip.StripTags(out)
+	out = keywordPattern.ReplaceAllLiteralString(out, "")
+	return strings.TrimSpace(out)
+}
+
+var keywordPattern = regexp.MustCompile(`((?:(?:[A-Z][a-z]+(\.|\s[0-9]+\.)\s*)+))`)
+
+func findKeywords(text string) string {
+	return strings.TrimSpace(keywordPattern.FindString(text))
 }

cmd/beornextract/types/card.go

@@ -1,33 +1,27 @@
 package types
 
 type HallOfBeornCard struct {
-	PackCode    string `json:"pack_code"`
-	PackName    string `json:"pack_name"`
-	IsOfficial  bool   `json:"is_official"`
-	TypeCode    string `json:"type_code"`
-	TypeName    string `json:"type_name"`
-	SphereCode  string `json:"sphere_code"`
-	SphereName  string `json:"sphere_name"`
-	Position    int    `json:"position"`
-	Code        string `json:"code"`
-	Name        string `json:"name"`
-	Traits      string `json:"traits"`
-	Text        string `json:"text"`
-	Flavor      string `json:"flavor"`
-	IsUnique    bool   `json:"is_unique"`
-	Threat      int    `json:"threat"`
-	Willpower   int    `json:"willpower"`
-	Attack      int    `json:"attack"`
-	Defense     int    `json:"defense"`
-	Health      int    `json:"health"`
-	Quantity    int    `json:"quantity"`
-	DeckLimit   int    `json:"deck_limit"`
-	Illustrator string `json:"illustrator"`
-	Octgnid     string `json:"octgnid"`
-	HasErrata   bool   `json:"has_errata"`
-	URL         string `json:"url"`
-	Imagesrc    string `json:"imagesrc"`
-}
+	PackCode      string `json:"pack_code,omitempty"`
+	PackName      string `json:"pack_name,omitempty"`
+	IsOfficial    bool   `json:"is_official,omitempty"`
+	TypeCode      string `json:"type_code,omitempty"`
+	TypeName      string `json:"type_name,omitempty"`
+	SphereCode    string `json:"sphere_code,omitempty"`
+	SphereName    string `json:"sphere_name,omitempty"`
+	Position      int    `json:"position,omitempty"`
+	Threat        int    `json:"threat,omitempty"`
+	Willpower     int    `json:"willpower,omitempty"`
+	Attack        int    `json:"attack,omitempty"`
+	Defense       int    `json:"defense,omitempty"`
+	Health        int    `json:"health,omitempty"`
+	Octgnid       string `json:"octgnid,omitempty"`
+	HasErrata     bool   `json:"has_errata,omitempty"`
+	URL           string `json:"url,omitempty"`
+	QuestPoints   string `json:"quest,omitempty"`
+	VictoryPoints int    `json:"victory,omitempty"`
+	Imagesrc      string `json:"imagesrc,omitempty"`
 
-type NormalizedCard struct {
+	EncounterSet   string `json:"encounter_set,omitempty"`
+	EngagementCost string `json:"engagement_cost,omitempty"`
+	ThreatStrength int    `json:"threat_strength,omitempty"`
 }

deps.bzl

@@ -30,6 +30,20 @@ def go_dependencies():
         sum = "h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw=",
         version = "v1.5.2",
     )
+    go_repository(
+        name = "com_github_grokify_html_strip_tags_go",
+        build_file_proto_mode = "disable_global",
+        importpath = "github.com/grokify/html-strip-tags-go",
+        sum = "h1:0fThFwLbW7P/kOiTBs03FsJSV9RM2M/Q/MOnCQxKMo0=",
+        version = "v0.0.1",
+    )
+    go_repository(
+        name = "com_github_jessevdk_go_flags",
+        build_file_proto_mode = "disable_global",
+        importpath = "github.com/jessevdk/go-flags",
+        sum = "h1:1jKYvbxEjfUl0fmqTCOfonvskHHXMjBySTLW4y9LFvc=",
+        version = "v1.5.0",
+    )
 
     go_repository(
         name = "com_github_mattn_go_sqlite3",

go.mod

@@ -6,5 +6,8 @@ require github.com/davecgh/go-spew v1.1.1
 
 require (
 	github.com/bazelbuild/rules_go v0.41.0 // indirect
+	github.com/grokify/html-strip-tags-go v0.0.1 // indirect
+	github.com/jessevdk/go-flags v1.5.0 // indirect
 	github.com/mattn/go-sqlite3 v1.14.17 // indirect
+	golang.org/x/sys v0.0.0-20210510120138-977fb7262007 // indirect
 )

go.sum

@@ -2,5 +2,12 @@ github.com/bazelbuild/rules_go v0.41.0 h1:JzlRxsFNhlX+g4drDRPhIaU5H5LnI978wdMJ0v
 github.com/bazelbuild/rules_go v0.41.0/go.mod h1:TMHmtfpvyfsxaqfL9WnahCsXMWDMICTw7XeK9yVb+YU=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/grokify/html-strip-tags-go v0.0.1 h1:0fThFwLbW7P/kOiTBs03FsJSV9RM2M/Q/MOnCQxKMo0=
+github.com/grokify/html-strip-tags-go v0.0.1/go.mod h1:2Su6romC5/1VXOQMaWL2yb618ARB8iVo6/DR99A6d78=
+github.com/jessevdk/go-flags v1.5.0 h1:1jKYvbxEjfUl0fmqTCOfonvskHHXMjBySTLW4y9LFvc=
+github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c2EWnYs/m4=
 github.com/mattn/go-sqlite3 v1.14.17 h1:mCRHCLDUBXgpKAqIKsaAaAsrAlbkeomtRFKXh2L6YIM=
 github.com/mattn/go-sqlite3 v1.14.17/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg=
+golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210510120138-977fb7262007 h1:gG67DSER+11cZvqIMb8S8bt0vZtiN6xWYARwirrOSfE=
+golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=