package utils import ( "encoding/json" "fmt" "log" "net/http" "os" "regexp" "strconv" "strings" "time" "whimsoc/vindn-backend/models" ) // odhText mirrors one entry of an ODH item's "texts" array (rel/type/value // triples holding description, teaser, price info, etc. in several // languages/formats). type odhText struct { Rel string `json:"rel"` Type string `json:"type"` Value string `json:"value"` } // odhExtras captures the ODH fields that don't map onto flat Event columns // by name (description/teaser live inside "texts", dates inside // "timeIntervals") and need extraction after the plain json.Unmarshal. type odhExtras struct { Texts []odhText `json:"texts"` TimeIntervals []struct { Start string `json:"start"` End string `json:"end"` } `json:"timeIntervals"` } var htmlTagPattern = regexp.MustCompile(`<[^>]*>`) // extractText returns the plain-text value of the first "texts" entry whose // rel matches one of the given candidates (checked in order), preferring a // text/plain variant over text/html and stripping tags if only html exists. func extractText(texts []odhText, rels ...string) string { var htmlFallback string for _, rel := range rels { for _, t := range texts { if t.Rel != rel || t.Value == "" { continue } if t.Type == "text/plain" { return t.Value } if htmlFallback == "" && t.Type == "text/html" { htmlFallback = t.Value } } } return strings.TrimSpace(htmlTagPattern.ReplaceAllString(htmlFallback, "")) } func FetchDataFromAPI2(url string) ([]models.Event, error) { resp, err := http.Get(url) if err != nil { log.Printf("Failed to fetch data: %v", err) return nil, err } defer resp.Body.Close() var events []models.Event if err := json.NewDecoder(resp.Body).Decode(&events); err != nil { log.Printf("Failed to decode JSON: %v", err) return nil, err } return events, nil } func FetchDataFromAPI() ([]models.Event, error) { fmt.Println("Starting to get odh files") //dataHubTypes := [5]string{"Tour", "POI", "Gastro", "Hotel", "Event"} dataHubTypes := [1]string{"Event"} licenseKey := os.Getenv("ODH_LICENCE_KEY") var events []models.Event for _, dataHubType := range dataHubTypes { queryOffset := 0 limit := 20 for { rawItems, err := getItemsRaw(dataHubType, licenseKey, queryOffset, limit) if err != nil { log.Fatalf("Failed to fetch items: %v", err) } if len(rawItems) == 0 { break } for _, rawItem := range rawItems { var event models.Event if err := json.Unmarshal(rawItem, &event); err != nil { log.Printf("Failed to decode JSON: %v", err) return nil, err } var extras odhExtras if err := json.Unmarshal(rawItem, &extras); err != nil { log.Printf("Failed to decode text/date extras for %s: %v", event.ExternalID, err) } else { event.Description = extractText(extras.Texts, "details", "open-data-nrw-detail") event.Teaser = extractText(extras.Texts, "teaser", "open-data-nrw-teaser") if len(extras.TimeIntervals) > 0 { event.StartDate = extras.TimeIntervals[0].Start event.EndDate = extras.TimeIntervals[0].End } } fmt.Printf("Successfully got event %s \"%s\"\n", event.ExternalID, event.Title) events = append(events, event) } queryOffset += limit } } return events, nil } // get all items of a single type func getItemsRaw(dataHubType string, licenseKey string, queryOffset int, limit int) ([]json.RawMessage, error) { req, err := http.NewRequest("GET", "https://meta.et4.de/rest.ashx/search/", nil) if err != nil { log.Print(err) os.Exit(1) } q := req.URL.Query() q.Add("experience", "open-data-nrw-tourismus") q.Add("type", dataHubType) q.Add("licenseKey", licenseKey) q.Add("q", "area:\"Nordrhein-Westfalen\"") q.Add("template", "ET2014A.json") q.Add("limit", strconv.Itoa(limit)) q.Add("offset", strconv.Itoa(queryOffset)) q.Add("sort", "id asc") req.URL.RawQuery = q.Encode() fmt.Println("Query: " + req.URL.String()) client := http.Client{ Timeout: 30 * time.Second, } resp, err := client.Do(req) if err != nil { fmt.Printf("client: error making http request: %s\n", err) os.Exit(1) } defer resp.Body.Close() var apiResp models.ApiResponseRaw if err := json.NewDecoder(resp.Body).Decode(&apiResp); err != nil { return nil, err } return apiResp.Items, nil }