---
title: "ETF Data Analysis"
format: 
  html:
    code-fold: true
    code-summary: "Show Code"
vignette: >
  %\VignetteIndexEntry{ETF Data Analysis}
  %\VignetteEngine{quarto::html}
  %\VignetteEncoding{UTF-8}
---

```{r}
#| label: setup
#| include: false
#| error: true

# Robust Setup for CI
data_loaded <- FALSE
universe <- NULL
metadata <- NULL
history <- NULL
history_summary <- NULL
snapshot_info <- list(generated_at = NULL, source = NULL)
is_ci <- identical(Sys.getenv("CI"), "true")

tryCatch({
  suppressPackageStartupMessages({
    library(etfdata)
    if (requireNamespace("ggplot2", quietly = TRUE)) library(ggplot2)
    if (requireNamespace("dplyr", quietly = TRUE)) library(dplyr)
    if (requireNamespace("targets", quietly = TRUE)) library(targets)
    if (requireNamespace("stringr", quietly = TRUE)) library(stringr)
  })

  # Strategy: Load snapshot from installed package (CI) or source (Local)
  snapshot_file <- "vignette_data.rds"
  data_path <- system.file("extdata", snapshot_file, package = "etfdata")
  
  if (data_path == "" || !file.exists(data_path)) {
    # Try relative source path (Local dev)
    data_path <- file.path("..", "inst", "extdata", snapshot_file)
  }

  if (file.exists(data_path)) {
    snap <- readRDS(data_path)
    universe <- snap$universe
    metadata <- snap$metadata
    history <- snap$history
    history_summary <- snap$history_summary
    snapshot_info$generated_at <- snap$generated_at
    snapshot_info$source <- snap$source
    data_loaded <- TRUE
  } else {
    # Robust fallback to targets
    store_path <- NULL
    possible_paths <- c("_targets", "../_targets", "../../_targets")
    for (p in possible_paths) {
      if (dir.exists(p)) {
        store_path <- p
        break
      }
    }
    
    if (!is.null(store_path) && requireNamespace("targets", quietly = TRUE)) {
      tar_config_set(store = store_path)
      try({
        universe <- tar_read(universe)
        metadata <- tar_read(metadata)
        history <- tar_read(history)
        tmp_summary <- try(tar_read(history_summary), silent = TRUE)
        if (!inherits(tmp_summary, "try-error")) {
          history_summary <- tmp_summary
        }
        snapshot_info$source <- "targets"
        data_loaded <- TRUE
      }, silent = TRUE)
    }
  }
}, error = function(e) {
  message("Setup failed: ", e$message)
})
```

## ETF Universe

We start by defining the universe of European UCITS ETFs listed on the LSE.
This universe is precomputed during CI to avoid browser CORS restrictions.

The logic used to retrieve the universe list is:

```{r universe-code}
#| echo: true
#| eval: false
get_etf_universe <- function(n = 20) {
  path <- system.file("extdata", "etf_universe.csv", package = "etfdata")
  if (path == "") {
    path <- "inst/extdata/etf_universe.csv"
  }
  if (file.exists(path)) {
    res <- readr::read_csv(path, show_col_types = FALSE)
    return(utils::head(res, n))
  }

  seed_path <- system.file("extdata", "seed_universe.csv", package = "etfdata")
  if (seed_path == "") {
    seed_path <- "inst/extdata/seed_universe.csv"
  }
  res <- readr::read_csv(seed_path, show_col_types = FALSE)
  utils::head(res, n)
}
```

```{r universe-table}
#| echo: false
#| eval: true
if (data_loaded && !is.null(universe)) {
  if (!is.null(snapshot_info$generated_at)) {
    message("Snapshot generated at: ", format(snapshot_info$generated_at))
  }
  knitr::kable(head(universe, 10))
  print(head(universe, 10))
} else {
  print("Universe data not available in this build.")
}
```

## Metadata (JustETF)

Metadata comes from the JustETF screener table (full universe), which includes
fund size and TER for LSE-listed UCITS. The pipeline uses the screener result
as the cached metadata snapshot.

```{r metadata-code}
#| echo: true
#| eval: false
metadata <- fetch_justetf_screener(min_aum_gbp = 0, max_ter = Inf)
```

```{r metadata-table}
#| echo: false
#| eval: true
if (data_loaded && !is.null(metadata)) {
  # Check if columns exist before selecting
  cols <- c("isin", "aum_text", "ter_text", "replication")
  cols <- intersect(cols, colnames(metadata))
  if (length(cols) > 0) {
    knitr::kable(head(metadata[, cols, drop=FALSE], 10))
    print(head(metadata, 10))
  }
} else {
  print("Metadata not available.")
}
```

## JustETF Screener API

In addition to the cached snapshot, we can query the JustETF screener directly
to find ETFs meeting specific criteria (e.g., AUM > £200m, TER < 0.75%). Browser
builds use the cached snapshot because cross-site requests are blocked by CORS.
The snapshot is built from a curated ETF universe list and incremental Yahoo
price downloads (only missing dates are fetched). See
`docs/wiki/ETF_Data_Sources.md` for details.

```{r screener-code}
#| echo: true
#| eval: false
screener_results <- fetch_justetf_screener(min_aum_gbp = 200, max_ter = 0.75)
print(head(screener_results))
```

## Price History (Yahoo Finance)

We fetch daily price history from Yahoo Finance using `quantmod`. For the
snapshot, we attempt a single download per ticker with a short timeout and
store the results for use in WebR/Shinylive.

```{r history-code}
#| echo: true
#| eval: false
fetch_price_history("VUSA.L")
```

```{r history-plot, echo=TRUE, message=FALSE, warning=FALSE}
is_ci <- isTRUE(get0("is_ci", ifnotfound = FALSE))
if (is_ci) {
  message("Skipping history plot in CI.")
} else if (data_loaded && !is.null(history) && requireNamespace("ggplot2", quietly = TRUE)) {
  # Visualize Close Prices with Facets
  print(
    history %>%
      ggplot(aes(x = date, y = close, color = ticker)) +
      geom_line() +
      facet_wrap(~ticker, scales = "free_y") +
      labs(title = "ETF Price History (LSE)", y = "Close Price (GBP)") +
      theme_minimal() +
      theme(legend.position = "none")
  )
  print(history)
  if (!is.null(history_summary)) {
    print(head(history_summary, 10))
  }
}
```

```{r history-plot-static, echo=FALSE}
# Static fallback for CI
if (isTRUE(get0("is_ci", ifnotfound = FALSE))) {
  knitr::include_graphics("history_plot.png")
}
```

## Combined Analysis

We join the datasets and parse the AUM strings to analyze the relationship between Fund Size, Liquidity, and Fees.

```{r combined, echo=TRUE, message=FALSE, warning=FALSE}
if (data_loaded && !is.null(history) && !is.null(metadata)) {
  avg_vol <- history %>%
    group_by(ticker) %>%
    summarise(avg_daily_vol = mean(volume, na.rm = TRUE))
  
  if ("isin" %in% colnames(universe) && "isin" %in% colnames(metadata)) {
    combined <- universe %>%
      inner_join(metadata, by = "isin") 

    if (!"ticker" %in% names(combined)) {
      ticker_cols <- intersect(c("ticker", "ticker.x", "ticker.y", "ticker_meta"), names(combined))
      if (length(ticker_cols) > 0) {
        combined <- combined %>%
          dplyr::mutate(
            ticker = dplyr::coalesce(!!!rlang::syms(ticker_cols))
          ) %>%
          dplyr::select(-dplyr::any_of(setdiff(ticker_cols, "ticker")))
      }
    }

    combined <- combined %>%
      inner_join(avg_vol, by = "ticker")
    
    if (requireNamespace("stringr", quietly = TRUE) && requireNamespace("dplyr", quietly = TRUE)) {
      if ("aum_text" %in% names(combined) && any(!is.na(combined$aum_text))) {
        combined <- bind_cols(combined, parse_aum(combined$aum_text))
      } else {
        combined <- combined %>%
          dplyr::mutate(
            aum_amount = NA_real_,
            aum_units = NA_character_,
            total_amount = NA_real_
          )
      }
    }

    if (requireNamespace("readr", quietly = TRUE) && requireNamespace("stringr", quietly = TRUE)) {
      if ("ter_text" %in% names(combined)) {
        combined <- combined %>%
          mutate(
            ter_val = readr::parse_number(ter_text),
            ter_units = stringr::str_extract(ter_text, "[%]") 
          )
      } else {
        combined <- combined %>%
          mutate(
            ter_val = NA_real_,
            ter_units = NA_character_
          )
      }
    }

    cols_to_show <- c("ticker", "name", "currency", "aum_amount", "aum_units", "avg_daily_vol", "ter_val")
    cols_to_show <- intersect(cols_to_show, colnames(combined))
    knitr::kable(head(combined %>% select(all_of(cols_to_show))))
  }
} else {
  print("Combined data not available for this build.")
}
```

```{r combined-plot-liquidity, echo=TRUE, message=FALSE, warning=FALSE}
if (exists("combined") && requireNamespace("ggplot2", quietly = TRUE)) {
  print(
    ggplot(combined, aes(x = total_amount, y = avg_daily_vol)) +
      geom_point() +
      scale_x_log10(labels = scales::label_currency(prefix = "£", scale_cut = scales::cut_short_scale())) +
      scale_y_log10(labels = scales::label_number(scale_cut = scales::cut_short_scale())) +
      labs(title = "ETF Size vs Liquidity",
           x = "Assets Under Management (AUM in GBP)",
           y = "Average Daily Volume") +
      theme_minimal()
  )
}
```

```{r combined-plot-ter, echo=TRUE, message=FALSE, warning=FALSE}
if (exists("combined") && "ter_val" %in% colnames(combined) && requireNamespace("ggplot2", quietly = TRUE)) {
  print(
    ggplot(combined, aes(x = total_amount, y = ter_val)) +
      geom_point() +
      scale_x_log10(labels = scales::label_currency(prefix = "£", scale_cut = scales::cut_short_scale())) +
      labs(title = "ETF Size vs Cost (TER)",
           x = "Assets Under Management (AUM in GBP)",
           y = "Total Expense Ratio (%)") +
      theme_minimal()
  )
} else if (exists("combined")) {
  print("TER data not available for plotting (ter_val column missing).")
} else {
  print("Combined data not available for plotting.")
}
```

```{r combined-plot-static, echo=FALSE}
if (!exists("combined") || !requireNamespace("ggplot2", quietly = TRUE)) {
  if (file.exists("combined_plot.png")) {
    knitr::include_graphics("combined_plot.png")
  } else {
    cat("Static plot not available for this build.\n")
  }
}
```

# Reproducibility & Metadata

## Session Info

<details>
<summary>Click to expand Session Info</summary>
```{r session-info}
sessionInfo()
```
</details>

## Data Structures

<details>
<summary>Click to expand Data Structures</summary>
```{r data-sizes}
if (data_loaded) {
  print("Universe:")
  print(head(universe, 10))
  print("Metadata:")
  print(head(metadata, 10))
  print("History:")
  print(head(history, 10))
}
```
</details>

## Targets Metadata

<details>
<summary>Click to expand Targets Metadata</summary>
```{r targets-meta, results='asis'}
#| echo: false
#| eval: true
#| message: false
#| warning: false
#| error: true

is_ci <- isTRUE(get0("is_ci", ifnotfound = FALSE))

if (requireNamespace("targets", quietly = TRUE) && 
    requireNamespace("visNetwork", quietly = TRUE)) {
  
  # Robust path finding for _targets and _targets.R
  store_path <- NULL
  possible_paths <- c("_targets", "../_targets", "../../_targets")
  for (p in possible_paths) {
    if (dir.exists(p)) {
      store_path <- p
      break
    }
  }
  
  script_path <- NULL
  possible_scripts <- c("_targets.R", "../_targets.R", "../../_targets.R")
  for (s in possible_scripts) {
    if (file.exists(s)) {
      script_path <- s
      break
    }
  }
  
  if (!is.null(store_path)) {
    # Set config for this session
    try(tar_config_set(store = store_path), silent = TRUE)
    if (!is.null(script_path)) {
      try(tar_config_set(script = script_path), silent = TRUE)
    }
    
    # 1. Meta Table
    try({
      if (!is.null(tar_meta(store = store_path))) {
        meta <- tar_meta(store = store_path)
        
        # Format bytes to human readable
        format_bytes <- function(b) {
          if (is.na(b)) return("NA")
          if (b < 1024) return(paste0(b, " B"))
          if (b < 1024^2) return(paste0(round(b / 1024, 1), " KB"))
          return(paste0(round(b / 1024^2, 1), " MB"))
        }
        
        meta$bytes_formatted <- sapply(meta$bytes, format_bytes)
        
        # Sort and select
        meta_disp <- meta[order(meta$bytes, decreasing = TRUE), ]
        meta_disp <- head(meta_disp, 5)
        meta_disp <- meta_disp[, c("name", "bytes_formatted", "seconds", "time", "type")]
        
        if (nrow(meta_disp) > 0) {
          print(knitr::kable(meta_disp, caption = "Targets Pipeline Metadata (Top 5)"))
        }
      }
    }, silent = TRUE)
    
    # 2. Pipeline Visualization
    cat("\n### Pipeline Network\n")
    if (!is_ci) {
      tryCatch({
        vn <- tar_visnetwork(store = store_path, targets_only = TRUE)
        print(vn)
      }, error = function(e) {
        message("Could not generate visNetwork: ", e$message)
      })
    } else {
      # Fallback: Print Manifest Table
      if (!is.null(script_path)) {
        try({
          man <- tar_manifest(fields = c("name", "command"))
          print(knitr::kable(head(man, 10), caption = "Pipeline Manifest (CI Fallback)"))
        })
      } else {
        message("Pipeline manifest skipped (script not found in CI).")
      }
    }
    
  } else {
    message("Targets store not found. Checked: ", paste(possible_paths, collapse = ", "))
  }
}
```
</details>
