---
title: "CY 2024 Report"
output:
  flexdashboard::flex_dashboard:
    orientation: rows
    vertical_layout: scroll
    source: embed
    smart: false
    theme: lumen
    navbar: 
      - { title: "NJIT HPC", href: "https://hpc.njit.edu"}
      - { title: "Daily Monitoring", href: "https://web.njit.edu/~abp/load.html"}
---

```{r setup, include=F}
library(flexdashboard)
library(tidyverse)
library(lubridate)
library(reshape2)
library(stringr)
library(plotly)
library(knitr)
library(DT)
options(DT.options = list(dom = 'ltip', pageLength = 25, lengthMenu = c(10, 25, 50, 75, 100), autoWidth = TRUE) )
library(kableExtra)
options(knitr.table.format = "html")
source("./ayfuncs.R")

```

Summary 
================================

```{r annual-setup,echo=F}
jobsthisay2024 <- read_delim('jobsalloc.csv.gz',delim=";") %>% filter(End >= as.Date("2024-01-01") & End < as.Date("2025-01-01"))
jobsthismonth <- read_delim('jobs-0.csv',delim=";")#, col_select=c(JobID:End)) 
day_of_month <- mday(today())
if ( day_of_month > 1 ){
  jobsthisay <- rbind(jobsthisay2024,jobsthismonth) %>%
     separate(Account, c("Account", "year1", "year2") , sep="_", fill = "right") %>%
     select(-c("year1","year2")) %>%
     mutate(Account = gsub("^([0-9]{4}-(spring|summer|fall)-[a-z]+-\\d+-[a-z0-9]+)-[a-z0-9]+$", "\\1", Account))
} else {
  jobsthisay <- jobsthisay2024 %>%
     mutate(Account = gsub("^([0-9]{4}-(spring|summer|fall)-[a-z]+-\\d+-[a-z0-9]+)-[a-z0-9]+$", "\\1", Account))
}
#	rbind(jobsthisay2021,jobsthisay2122) %>% filter(End >= as.Date("2021-01-01") & End < as.Date("2022-01-01"))

dayspermonth = c(31,29,31,30,31,30,31,31,30,31,30,31)
general <- rep(100,12)*128*dayspermonth*24
bigmem <- rep(2,12)*128*dayspermonth*24
gpusu <- rep(25,12)*128*dayspermonth*24
cy2024su <- general + bigmem + gpusu
if ( Sys.Date() > as.Date("2024-12-31")){
  currmonth <- length(seq(from=as.Date("2024-01-01"), to=as.Date("2024-12-31"), by='month'))
} else {
  if ( day_of_month > 1 ){
    currmonth <- length(seq(from=as.Date("2024-01-01"), to=Sys.Date(), by='month'))
  } else {
    currmonth <- length(seq(from=as.Date("2024-01-01"), to=Sys.Date()-1, by='month'))
  }
}
jobsthisay %>% 
  mutate(Year=year(End),Month=month(End,label=TRUE)) %>% 
  group_by(Year,Month) %>% 
  select(c(Month,Start,Submit,End,NCPUS))  %>% 
  summarize( 
    Total_SU=round(sum(as.numeric(difftime(End,Start,units="hours"))*NCPUS),2), 
    Total_Jobs=n() 
  ) -> percent
jobsthisay_trim <- jobsthisay %>%
  mutate(
    QOS = gsub("high_[a-z0-9]+", "high", QOS),
    Account = gsub("qm32","jyoung", Account)
    )
pis <- read_delim('pi.txt',delim="|") 
users <- read_delim('users.txt', delim="|") %>% select(c(Account:Name))
pi_users <- full_join(pis,users)
pi_users_alloc <- pi_users %>% group_by(Account) %>% summarize(Users=n_distinct(User))
pi_users_dept <- pi_users %>% group_by(Department) %>% summarize(PIs=n_distinct(Account),Users=n_distinct(User))
pi_users_college <- pi_users %>% group_by(College) %>% summarize(PIs=n_distinct(Account),Users=n_distinct(User))
```


Row {.tabset data-height=1200}
---------------------------------

### Summary

```{r alloc-summary,echo=F}
jobsthisay_trim %>% group_by(Account) %>% summary  -> Timing_Part
jobsthisay_trim %>% summary -> Timing_Total

full_join(full_join(pis,Timing_Part),pi_users_alloc) %>% drop_na -> Timing_Part
Timing_Total %>% mutate(Account="Total",'PI Name'="Wulver Cluster",Department="Research Computing",College="NJIT",Users=sum(Timing_Part$Users)) -> Timing_Total
full_join(Timing_Part,Timing_Total) -> Timing
#Timing[is.na(Timing)] <- "Total"

Timing %>% select(c(Account,'PI Name', Department, College, Users, SU:CPUs)) %>% 
  datatable(rownames = FALSE,options = list(autoWidth = FALSE,pageLength = 100,order = list(list(5, 'desc'))))
```

### Usage Summary by QOS

```{r alloc-usage-summary-qos,echo=F}
jobsthisay_trim %>% group_by(Account,QOS) %>% filter(str_detect(QOS,"standard")) %>% summary  -> std
jobsthisay_trim %>% group_by(Account,QOS) %>% filter(str_detect(QOS,"low")) %>% summary -> low
jobsthisay_trim %>% group_by(Account,QOS) %>% filter(str_detect(QOS,"high")) %>% summary -> high
  #mutate(QOS = gsub("high_[a-z0-9]+", "high", QOS)) %>%
jobsthisay_trim %>% group_by(Account,QOS) %>% filter(str_detect(QOS,"debug")) %>% summary -> debug
usage_summary <- full_join(full_join(full_join(high,std),low),debug) %>% select(c(Account,QOS,SU)) %>% pivot_wider(names_from = QOS, values_from = SU)
usage_summary[is.na(usage_summary)] <- 0
full_join(full_join(pis,usage_summary),pi_users_alloc) %>% drop_na() %>%
  mutate(Total=round(high+low+standard+debug,2)) %>% select(c(Account:College,Users,Total,Standard=standard,Low=low,High=high,Debug=debug)) %>%
  datatable(rownames = FALSE,options = list(autoWidth = FALSE,pageLength = 100,order = list(list(5, 'desc'))))
```

### Number of Jobs by QOS

```{r alloc-jobs-summary-qos,echo=F}
usage_summary <- full_join(full_join(full_join(high,std),low),debug) %>% select(c(Account,QOS,Jobs)) %>% pivot_wider(names_from = QOS, values_from = Jobs)
usage_summary[is.na(usage_summary)] <- 0
full_join(full_join(pis,usage_summary),pi_users_alloc) %>% drop_na() %>%
  mutate(Total=round(high+low+standard+debug,2)) %>% select(c(Account:College,Users,Total,Standard=standard,Low=low,High=high,Debug=debug)) %>%
  datatable(rownames = FALSE,options = list(autoWidth = FALSE,pageLength = 100,order = list(list(5, 'desc'))))
```


### Usage Summary by Department

```{r alloc-summary-su-department,echo=F}
jobsthisay_trim %>% group_by(Account,QOS) %>% summary %>%
   select(c(Account,QOS,SU,Jobs)) %>% pivot_wider(names_from = QOS, values_from = c(SU,Jobs)) %>% select(-c(SU_test,Jobs_test)) -> Timing_Part
Timing_Part[is.na(Timing_Part)] <- 0
jobsthisay_trim %>% summary -> Timing_Total

full_join(pis,Timing_Part) %>% drop_na %>%
  mutate(SU=round(SU_standard+SU_low+SU_debug+SU_high,2),Jobs=Jobs_standard+Jobs_low+Jobs_debug+Jobs_high) -> Timing

Timing %>% group_by(Department) %>% summarize(PIs=n(),
  SU=round(sum(SU),2),
  Standard=round(sum(SU_standard),2),
  Low=round(sum(SU_low),2),
  High=round(sum(SU_high),2),
  Debug=round(sum(SU_debug),2)
  ) %>%
  full_join(pi_users_dept) %>% drop_na() %>% select(c(Department,PIs,Users,SU:Debug)) %>%
  datatable(rownames = FALSE, ,options = list(autoWidth = FALSE,pageLength = 100,order = list(list(3, 'desc'))))
```


### Number of Jobs by Department

```{r alloc-summary-jobs-department,echo=F}
Timing %>% group_by(Department) %>% summarize(PIs=n(),
  Jobs=sum(Jobs),
  Standard=sum(Jobs_standard),
  Low=sum(Jobs_low),
  High=sum(Jobs_high),
  Debug=sum(Jobs_debug)
  ) %>%
  full_join(pi_users_dept) %>% drop_na() %>% select(c(Department,PIs,Users,Jobs:Debug)) %>%
  datatable(rownames = FALSE, ,options = list(autoWidth = FALSE,pageLength = 100,order = list(list(3, 'desc'))))
```

### Usage Summary by College

```{r alloc-summary-su-college,echo=F}
Timing %>% group_by(College) %>% summarize(PIs=n(),
  SU=round(sum(SU),2),
  Standard=round(sum(SU_standard),2),
  Low=round(sum(SU_low),2),
  High=round(sum(SU_high),2),
  Debug=round(sum(SU_debug),2)
  ) %>%
  full_join(pi_users_college) %>% drop_na() %>% select(c(College,PIs,Users,SU:Debug)) %>%
  datatable(rownames = FALSE,options = list(autoWidth = FALSE,pageLength = 100,order = list(list(3, 'desc'))))
```

### Number of Jobs by College

```{r alloc-summary-jobs-college,echo=F}
Timing %>% group_by(College) %>% summarize(PIs=n(),
  Jobs=sum(Jobs),
  Standard=sum(Jobs_standard),
  Low=sum(Jobs_low),
  High=sum(Jobs_high),
  Debug=sum(Jobs_debug)
  ) %>%
  full_join(pi_users_college) %>% drop_na() %>% select(c(College,PIs,Users,Jobs:Debug)) %>%
  datatable(rownames = FALSE,options = list(autoWidth = FALSE,pageLength = 100,order = list(list(3, 'desc'))))
```

Summary {data-navmenu="Annual Report"}
==============================

Row
-------------------------------------



```{r annualsummary}
jobsthisay_trim %>% summary %>% 
   mutate(Partition="Total",Used=round(SU/sum(cy2024su[1:currmonth])*100,2)) %>%
   select(c(Partition,SU,'% Consumed'=Used,Jobs,Wait,Run)) -> Total_SU_Jobs
jobsthisay_trim %>% serial_filter %>% summary %>% 
  mutate(Partition="Total",
         SUper = round(SU/Total_SU_Jobs$SU*100,2), 
         Jobsper = round(Jobs/Total_SU_Jobs$Jobs*100,2)) %>%
  select(c(Partition,SU,Jobs,'% SU'=SUper,'% Jobs'=Jobsper,Wait,Run)) -> serialtotal
jobsthisay_trim %>% smp_filter %>% summary %>% 
  mutate(Partition="Total",
         SUper = round(SU/Total_SU_Jobs$SU*100,2), 
         Jobsper = round(Jobs/Total_SU_Jobs$Jobs*100,2)) %>%
  select(c(Partition,SU,Jobs,'% SU'=SUper,'% Jobs'=Jobsper,Wait,Run)) -> smptotal
jobsthisay_trim %>% dmp_filter %>% summary %>% 
  mutate(Partition="Total",
         SUper = round(SU/Total_SU_Jobs$SU*100,2), 
         Jobsper = round(Jobs/Total_SU_Jobs$Jobs*100,2)) %>%
  select(c(Partition,SU,Jobs,'% SU'=SUper,'% Jobs'=Jobsper,Wait,Run)) -> dmptotal
```

### Total

```{r annualsummary-total}
jobsthisay_trim %>% filter(str_detect(Partition,'general') | str_detect(Partition,'test') | str_detect(Partition,'debug')) %>% summary %>% partfilter('general',general) -> generaltotal
jobsthisay_trim %>% filter(str_detect(Partition,'bigmem')) %>% summary %>% partfilter('bigmem',bigmem) -> bigmemtotal
jobsthisay_trim %>% filter(str_detect(Partition,'gpu')) %>% summary %>% partfilter('gpu',gpusu) -> gputotal
rbind(Total_SU_Jobs,generaltotal,bigmemtotal,gputotal) %>%
  kable() %>% kable_styling(bootstrap_options = c("striped", "hover", "bordered", "condensed", "responsive") )
```

### CPU vs GPU

```{r annualcpuvgpu-summary,echo=F}
jobsthisay_trim %>% summary -> Timing_Total
jobsthisay_trim %>% filter(!str_detect(Partition,"gpu")) %>% summary %>% 
   mutate(Type="cpu","% SU"=round(SU/Timing_Total$SU*100,2),"% Jobs"=round(Jobs/Timing_Total$Jobs*100,2)) %>% 
   select(c("Type","SU","% SU","Jobs","% Jobs","Wait","Run")) -> cpu
jobsthisay_trim %>% filter(str_detect(Partition,"gpu")) %>% summary %>% 
   mutate(Type="gpu","% SU"=round(SU/Timing_Total$SU*100,2),"% Jobs"=round(Jobs/Timing_Total$Jobs*100,2)) %>% 
   select(c("Type","SU","% SU","Jobs","% Jobs","Wait","Run")) -> gpu
jobsthisay_trim %>% summary %>% 
   mutate(Type="Total","% SU"=round(SU/Timing_Total$SU*100,2),"% Jobs"=round(Jobs/Timing_Total$Jobs*100,2)) %>% 
   select(c("Type","SU","% SU","Jobs","% Jobs","Wait","Run")) -> total
full_join(full_join(cpu,gpu),total) %>%
  kable() %>% kable_styling(bootstrap_options = c("striped", "hover", "bordered", "condensed", "responsive") )
```


Row
-----------------------
### Serial Jobs

```{r annualsummary-serial}
jobsthisay_trim %>% filter(str_detect(Partition,'general') | str_detect(Partition,'test') | str_detect(Partition,'debug')) %>% 
  serial_filter %>% summary %>% parttotalfilter('general',generaltotal) -> general_serial
jobsthisay_trim %>% filter(str_detect(Partition,'bigmem')) %>% 
  serial_filter %>% summary %>% parttotalfilter('bigmem',bigmemtotal) -> bigmem_serial
jobsthisay_trim %>% filter(str_detect(Partition,'gpu')) %>% 
  serial_filter %>% summary %>% parttotalfilter('gpu',gputotal) -> gpu_serial
rbind(serialtotal,general_serial,bigmem_serial,gpu_serial) %>%
  kable() %>% kable_styling(bootstrap_options = c("striped", "hover", "bordered", "condensed", "responsive") )
```

### SMP Jobs

```{r annualsummary-smp}
jobsthisay_trim %>% filter(str_detect(Partition,'general') | str_detect(Partition,'test') | str_detect(Partition,'debug')) %>% 
  smp_filter %>% summary %>% parttotalfilter('general',generaltotal) -> general_smp
jobsthisay_trim %>% filter(str_detect(Partition,'bigmem')) %>% 
  smp_filter %>% summary %>% parttotalfilter('bigmem',bigmemtotal) -> bigmem_smp
jobsthisay_trim %>% filter(str_detect(Partition,'gpu')) %>% 
  smp_filter %>% summary %>% parttotalfilter('gpu',gputotal) -> gpu_smp
rbind(smptotal,general_smp,bigmem_smp,gpu_smp) %>%
  kable() %>% kable_styling(bootstrap_options = c("striped", "hover", "bordered", "condensed", "responsive") )
```

### DMP Jobs

```{r annualsummary-dmp}
jobsthisay_trim %>% filter(str_detect(Partition,'general') | str_detect(Partition,'test') | str_detect(Partition,'debug')) %>% 
  dmp_filter %>% summary %>% parttotalfilter('general',generaltotal) -> general_dmp
jobsthisay_trim %>% filter(str_detect(Partition,'bigmem')) %>% 
  dmp_filter %>% summary %>% parttotalfilter('bigmem',bigmemtotal) -> bigmem_dmp
jobsthisay_trim %>% filter(str_detect(Partition,'gpu')) %>% 
  dmp_filter %>% summary %>% parttotalfilter('gpu',gputotal) -> gpu_dmp
rbind(dmptotal,general_dmp,bigmem_dmp,gpu_dmp) %>%
  kable() %>% kable_styling(bootstrap_options = c("striped", "hover", "bordered", "condensed", "responsive") )
```


Row
--------------------------------------------------

### Serial Jobs

```{r annualcpuvgpu-serial,echo=F}
jobsthisay_trim %>% serial_filter %>% filter(!str_detect(Partition,"gpu")) %>% summary %>% 
   mutate(Type="cpu","% SU"=round(SU/Timing_Total$SU*100,2),"% Jobs"=round(Jobs/Timing_Total$Jobs*100,2)) %>% 
   select(c("Type","SU","% SU","Jobs","% Jobs","Wait","Run")) -> cpu
jobsthisay_trim %>% serial_filter %>% filter(str_detect(Partition,"gpu")) %>% summary %>% 
   mutate(Type="gpu","% SU"=round(SU/Timing_Total$SU*100,2),"% Jobs"=round(Jobs/Timing_Total$Jobs*100,2)) %>% 
   select(c("Type","SU","% SU","Jobs","% Jobs","Wait","Run")) -> gpu
jobsthisay_trim %>% serial_filter %>% summary %>% 
   mutate(Type="Total","% SU"=round(SU/Timing_Total$SU*100,2),"% Jobs"=round(Jobs/Timing_Total$Jobs*100,2)) %>% 
   select(c("Type","SU","% SU","Jobs","% Jobs","Wait","Run")) -> total
full_join(full_join(cpu,gpu),total) %>%
   kable() %>% kable_styling(bootstrap_options = c("striped", "hover", "bordered", "condensed", "responsive") )
```

### SMP Jobs

```{r annualcpuvgpu-smp,echo=F}
jobsthisay_trim %>% smp_filter %>% filter(!str_detect(Partition,"gpu")) %>% summary %>% 
   mutate(Type="cpu","% SU"=round(SU/Timing_Total$SU*100,2),"% Jobs"=round(Jobs/Timing_Total$Jobs*100,2)) %>% 
   select(c("Type","SU","% SU","Jobs","% Jobs","Wait","Run")) -> cpu
jobsthisay_trim %>% smp_filter %>% filter(str_detect(Partition,"gpu")) %>% summary %>% 
   mutate(Type="gpu","% SU"=round(SU/Timing_Total$SU*100,2),"% Jobs"=round(Jobs/Timing_Total$Jobs*100,2)) %>% 
   select(c("Type","SU","% SU","Jobs","% Jobs","Wait","Run")) -> gpu
jobsthisay_trim %>% smp_filter %>% summary %>% 
   mutate(Type="Total","% SU"=round(SU/Timing_Total$SU*100,2),"% Jobs"=round(Jobs/Timing_Total$Jobs*100,2)) %>% 
   select(c("Type","SU","% SU","Jobs","% Jobs","Wait","Run")) -> total
full_join(full_join(cpu,gpu),total) %>%
   kable() %>% kable_styling(bootstrap_options = c("striped", "hover", "bordered", "condensed", "responsive") )
```

### DMP Jobs

```{r annualcpuvgpu-dmp,echo=F}
jobsthisay_trim %>% dmp_filter %>% filter(!str_detect(Partition,"gpu")) %>% summary %>% 
   mutate(Type="cpu","% SU"=round(SU/Timing_Total$SU*100,2),"% Jobs"=round(Jobs/Timing_Total$Jobs*100,2)) %>% 
   select(c("Type","SU","% SU","Jobs","% Jobs","Wait","Run")) -> cpu
jobsthisay_trim %>% dmp_filter %>% filter(str_detect(Partition,"gpu")) %>% summary %>% 
   mutate(Type="gpu","% SU"=round(SU/Timing_Total$SU*100,2),"% Jobs"=round(Jobs/Timing_Total$Jobs*100,2)) %>% 
   select(c("Type","SU","% SU","Jobs","% Jobs","Wait","Run")) -> gpu
jobsthisay_trim %>% dmp_filter %>% summary %>% 
   mutate(Type="Total","% SU"=round(SU/Timing_Total$SU*100,2),"% Jobs"=round(Jobs/Timing_Total$Jobs*100,2)) %>% 
   select(c("Type","SU","% SU","Jobs","% Jobs","Wait","Run")) -> total
full_join(full_join(cpu,gpu),total) %>%
   kable() %>% kable_styling(bootstrap_options = c("striped", "hover", "bordered", "condensed", "responsive") )
```

Partition {data-navmenu="Annual Report"}
==============================


Row
---------------------------------

### Summary

```{r annualpart-summary,echo=F}
jobsthisay_trim %>% group_by(Partition) %>% summary -> Timing_Part
jobsthisay_trim %>% summary -> Timing_Total

full_join(Timing_Part,Timing_Total) %>% mutate('% SU' = round(SU/Timing_Total$SU*100,2)) %>% select(c('Partition','SU','% SU','Jobs', 'Wait', 'Run')) -> Timing
Timing[is.na(Timing)] <- "Total"

Timing %>% kable() %>% kable_styling(bootstrap_options = c("striped", "hover", "bordered", "condensed", "responsive") )
```


### Serial Jobs

```{r annualpart-serial,echo=F}
jobsthisay_trim %>% serial_filter %>% group_by(Partition) %>% summary  -> Timing_Part
jobsthisay_trim %>% serial_filter %>% summary -> Timing_Total

full_join(Timing_Part,Timing_Total) -> Timing
Timing[is.na(Timing)] <- "Total"

Timing %>% kable() %>% kable_styling(bootstrap_options = c("striped", "hover", "bordered", "condensed", "responsive") )
```

### SMP Jobs

```{r annualpart-smp,echo=F}
jobsthisay_trim %>% smp_filter %>% group_by(Partition) %>% summary  -> Timing_Part
jobsthisay_trim %>% smp_filter %>% summary -> Timing_Total

full_join(Timing_Part,Timing_Total) -> Timing
Timing[is.na(Timing)] <- "Total"

Timing %>% kable() %>% kable_styling(bootstrap_options = c("striped", "hover", "bordered", "condensed", "responsive") )
```

### DMP Jobs

```{r annualpart-dmp,echo=F}
jobsthisay_trim %>% dmp_filter %>% group_by(Partition) %>% summary  -> Timing_Part
jobsthisay_trim %>% dmp_filter %>% summary -> Timing_Total

full_join(Timing_Part,Timing_Total) -> Timing
Timing[is.na(Timing)] <- "Total"

Timing %>% kable() %>% kable_styling(bootstrap_options = c("striped", "hover", "bordered", "condensed", "responsive") )
```


QOS {data-navmenu="Annual Report"}
==============================


Row
---------------------------------

### Summary

```{r annualqos-summary,echo=F}
jobsthisay_trim %>% group_by(QOS) %>% summary -> Timing_Part
jobsthisay_trim %>% summary -> Timing_Total

full_join(Timing_Part,Timing_Total) %>% mutate('% SU' = round(SU/Timing_Total$SU*100,2)) %>% select(c('QOS','SU','% SU','Jobs', 'Wait', 'Run')) -> Timing
Timing[is.na(Timing)] <- "Total"

Timing %>% kable() %>% kable_styling(bootstrap_options = c("striped", "hover", "bordered", "condensed", "responsive") )
```


### Serial Jobs

```{r annualqos-serial,echo=F}
jobsthisay_trim %>% serial_filter %>% group_by(QOS) %>% summary  -> Timing_Part
jobsthisay_trim %>% serial_filter %>% summary -> Timing_Total

full_join(Timing_Part,Timing_Total) -> Timing
Timing[is.na(Timing)] <- "Total"

Timing %>% kable() %>% kable_styling(bootstrap_options = c("striped", "hover", "bordered", "condensed", "responsive") )
```

### SMP Jobs

```{r annualqos-smp,echo=F}
jobsthisay_trim %>% smp_filter %>% group_by(QOS) %>% summary  -> Timing_Part
jobsthisay_trim %>% smp_filter %>% summary -> Timing_Total

full_join(Timing_Part,Timing_Total) -> Timing
Timing[is.na(Timing)] <- "Total"

Timing %>% kable() %>% kable_styling(bootstrap_options = c("striped", "hover", "bordered", "condensed", "responsive") )
```

### DMP Jobs

```{r annualqos-dmp,echo=F}
jobsthisay_trim %>% dmp_filter %>% group_by(QOS) %>% summary  -> Timing_Part
jobsthisay_trim %>% dmp_filter %>% summary -> Timing_Total

full_join(Timing_Part,Timing_Total) -> Timing
Timing[is.na(Timing)] <- "Total"

Timing %>% kable() %>% kable_styling(bootstrap_options = c("striped", "hover", "bordered", "condensed", "responsive") )
```


Allocation {data-navmenu="Annual Report"}
==============================

Row
---------------------------------

### Summary

```{r annualalloc-summary,echo=F}
jobsthisay_trim %>% group_by(Account) %>% summary  -> Timing_Part
jobsthisay_trim %>% summary -> Timing_Total

full_join(Timing_Part,Timing_Total) -> Timing
Timing[is.na(Timing)] <- "Total"

Timing %>% my_table
```

### CPU vs GPU

```{r annualalloc-type,echo=F}
#jobsthisay_trim %>% cpu_gpu_total(Account) -> total
#jobsthisay_trim %>% cpu_total(Account) -> cpu
#jobsthisay_trim %>% gpu_total(Account) -> gpu
jobsthisay_trim %>% group_by(Account) %>% summary %>% select(c(Account,Total=SU)) -> total
jobsthisay_trim %>% filter(!str_detect(Partition,"gpu")) %>% group_by(Account) %>% summary %>% select(c(Account,CPU=SU)) -> cpu
jobsthisay_trim %>% filter(str_detect(Partition,"gpu")) %>% group_by(Account) %>% summary %>% select(c(Account,GPU=SU))  -> gpu
full_join(total,full_join(cpu,gpu)) -> usage1920
usage1920[is.na(usage1920)] <- 0
usage1920 %>% mutate('%CPU'=round(CPU/Total*100,2), '%GPU'=round(GPU/Total*100,2)) %>% my_table
```

Row
----------------------------------


### Serial Jobs

```{r annualalloc-serial,echo=F}
jobsthisay_trim %>% serial_filter %>% group_by(Account) %>% summary  -> Timing_Part
jobsthisay_trim %>% serial_filter %>% summary -> Timing_Total

full_join(Timing_Part,Timing_Total) -> Timing
Timing[is.na(Timing)] <- "Total"

Timing %>% my_table 
```

### SMP Jobs

```{r annualalloc-smp,echo=F}
jobsthisay_trim %>% smp_filter %>% group_by(Account) %>% summary  -> Timing_Part
jobsthisay_trim %>% smp_filter %>% summary -> Timing_Total

full_join(Timing_Part,Timing_Total) -> Timing
Timing[is.na(Timing)] <- "Total"

Timing %>% my_table
```

### DMP Jobs

```{r annualalloc-dmp,echo=F}
jobsthisay_trim %>% dmp_filter %>% group_by(Account) %>% summary  -> Timing_Part
jobsthisay_trim %>% dmp_filter %>% summary -> Timing_Total

full_join(Timing_Part,Timing_Total) -> Timing
Timing[is.na(Timing)] <- "Total"

Timing %>% my_table 
```

Row
------------------------------

### Standard

```{r annualalloc-std,echo=F}
jobsthisay_trim %>% filter(str_detect(QOS,'standard')) %>% group_by(Account) %>% summary  -> Timing_Part
jobsthisay_trim %>% filter(str_detect(QOS,'standard')) %>% summary -> Timing_Total

full_join(Timing_Part,Timing_Total) -> Timing
Timing[is.na(Timing)] <- "Total"

Timing %>% my_table 
```

### Low

```{r annualalloc-low,echo=F}
jobsthisay_trim %>% filter(str_detect(QOS,'low')) %>% group_by(Account) %>% summary  -> Timing_Part
jobsthisay_trim %>% filter(str_detect(QOS,'low')) %>% summary -> Timing_Total

full_join(Timing_Part,Timing_Total) -> Timing
Timing[is.na(Timing)] <- "Total"

Timing %>% my_table 
```

### High

```{r annualalloc-high,echo=F}
jobsthisay_trim %>% filter(str_detect(QOS,'high')) %>% group_by(Account) %>% summary  -> Timing_Part
jobsthisay_trim %>% filter(str_detect(QOS,'high')) %>% summary -> Timing_Total

full_join(Timing_Part,Timing_Total) -> Timing
Timing[is.na(Timing)] <- "Total"

Timing %>% my_table 
```

User {data-navmenu="Annual Report"}
==============================

Row
---------------------------------

### Summary

```{r annualuser-summary}
jobsthisay_trim %>% group_by(User) %>% summary  -> Timing_Part
jobsthisay_trim %>% summary -> Timing_Total

full_join(Timing_Part,Timing_Total) -> Timing
Timing[is.na(Timing)] <- "Total"

Timing %>% my_table
```

### CPU vs GPU

```{r annualuser-type,echo=F}
#jobsthisay_trim %>% cpu_gpu_total(User) -> total
#jobsthisay_trim %>% cpu_total(User) -> cpu
#jobsthisay_trim %>% gpu_total(User) -> gpu
jobsthisay_trim %>% group_by(User) %>% summary %>% select(c(User,Total=SU)) -> total
jobsthisay_trim %>% filter(!str_detect(Partition,"gpu")) %>% group_by(User) %>% summary %>% select(c(User,CPU=SU)) -> cpu
jobsthisay_trim %>% filter(str_detect(Partition,"gpu")) %>% group_by(User) %>% summary %>% select(c(User,GPU=SU))  -> gpu
full_join(total,full_join(cpu,gpu)) -> usage1920
usage1920[is.na(usage1920)] <- 0
usage1920 %>% mutate('%CPU'=round(CPU/Total*100,2), '%GPU'=round(GPU/Total*100,2)) %>% my_table
```

Row
--------------------------------------

### Serial

```{r annualuser-serial}
jobsthisay_trim %>% serial_filter %>% group_by(User) %>% summary  -> Timing_Part
jobsthisay_trim %>% serial_filter %>% summary -> Timing_Total

full_join(Timing_Part,Timing_Total) -> Timing
Timing[is.na(Timing)] <- "Total"

Timing %>% my_table 
```

### SMP

```{r annualuser-smp}
jobsthisay_trim %>% smp_filter %>% group_by(User) %>% summary  -> Timing_Part
jobsthisay_trim %>% smp_filter %>% summary -> Timing_Total

full_join(Timing_Part,Timing_Total) -> Timing
Timing[is.na(Timing)] <- "Total"

Timing %>% my_table
```

### DMP

```{r annualuser-dmp}
jobsthisay_trim %>% dmp_filter %>% group_by(User) %>% summary  -> Timing_Part
jobsthisay_trim %>% dmp_filter %>% summary -> Timing_Total

full_join(Timing_Part,Timing_Total) -> Timing
Timing[is.na(Timing)] <- "Total"

Timing %>% my_table
```

Row
----------------------------------

### Standard

```{r annualuser-std}
jobsthisay_trim %>% filter(str_detect(QOS,'standard')) %>% group_by(User) %>% summary  -> Timing_Part
jobsthisay_trim %>% filter(str_detect(QOS,'standard')) %>% summary -> Timing_Total

full_join(Timing_Part,Timing_Total) -> Timing
Timing[is.na(Timing)] <- "Total"

Timing %>% my_table 
```

### Low

```{r annualuser-low}
jobsthisay_trim %>% filter(str_detect(QOS,'low')) %>% group_by(User) %>% summary  -> Timing_Part
jobsthisay_trim %>% filter(str_detect(QOS,'low')) %>% summary -> Timing_Total

full_join(Timing_Part,Timing_Total) -> Timing
Timing[is.na(Timing)] <- "Total"

Timing %>% my_table 
```

### High

```{r annualuser-high}
jobsthisay_trim %>% filter(str_detect(QOS,'high')) %>% group_by(User) %>% summary  -> Timing_Part
jobsthisay_trim %>% filter(str_detect(QOS,'high')) %>% summary -> Timing_Total

full_join(Timing_Part,Timing_Total) -> Timing
Timing[is.na(Timing)] <- "Total"

Timing %>% my_table 
```





Summary {data-navmenu="Monthly Reports"}
=========================================

Row 
------------------------------------

### Summary

```{r monthly-setup,echo=F}
jobsthisay %>% monthly_summary %>% partpercent(cy2024su[1:currmonth]) -> total
jobsthisay %>% serial_filter %>% monthly_summary -> serial
jobsthisay %>% smp_filter %>% monthly_summary -> smp
jobsthisay %>% dmp_filter %>% monthly_summary -> dmp

getpercent(serial,percent) -> serialpercent
getpercent(smp,percent) -> smppercent
getpercent(dmp,percent) -> dmppercent

#cbind(total,serialpercent,smppercent,dmppercent)  %>% select(-c(Year1,Month1,Year2,Month2,Year3,Month3)) -> jobtotal
#colnames(jobtotal) <- c("Year","Month","SU","Jobs","Wait","Run","SU","Jobs","% SU","%Jobs","Wait","Run","SU","Jobs","% SU","%Jobs","Wait","Run","SU","Jobs","% SU","%Jobs","Wait","Run")
#jobtotal %>% 
#  kable() %>%
#  kable_styling(bootstrap_options = c("striped", "hover", "bordered", "condensed", "responsive")) %>%
#  add_header_above(c(" " = 2, "Total" = 4, "Serial" = 6, "SMP" = 6, "DMP" = 6))
```


```{r monthly-summary}
total %>%
  kable() %>% kable_styling(bootstrap_options = c("striped", "hover", "bordered", "condensed", "responsive") )
```

### CPU vs GPU

```{r monthly-summarytype, echo=F}
#jobsthisay %>% cpu_gpu_monthly -> total
#jobsthisay %>% cpu_monthly -> cpu
#jobsthisay %>% gpu_monthly -> gpu
jobsthisay_trim %>% monthly_summary %>% select(c(Year,Month,Total=SU)) -> total
jobsthisay_trim %>% filter(!str_detect(Partition,"gpu")) %>% monthly_summary %>% select(c(Year,Month,CPU=SU)) -> cpu
jobsthisay_trim %>% filter(str_detect(Partition,"gpu")) %>% monthly_summary %>% select(c(Year,Month,GPU=SU)) -> gpu 
full_join(total,full_join(cpu,gpu)) %>% mutate('%CPU'=round(CPU/Total*100,2), '%GPU'=round(GPU/Total*100,2)) -> usage1920
usage1920 %>% kable() %>% kable_styling(bootstrap_options = c("striped", "hover", "bordered", "condensed", "responsive") )
```

### Serial Jobs

```{r monthly-serial}
serialpercent %>%
  kable() %>% kable_styling(bootstrap_options = c("striped", "hover", "bordered", "condensed", "responsive") )
```

### SMP Jobs

```{r monthly-smp}
smppercent %>% 
  kable() %>% kable_styling(bootstrap_options = c("striped", "hover", "bordered", "condensed", "responsive") )
```

### DMP Jobs

```{r monthly-dmp}
dmppercent %>% 
  kable() %>% kable_styling(bootstrap_options = c("striped", "hover", "bordered", "condensed", "responsive") )
```


Partition {data-navmenu="Monthly Reports"}
===============================================

Row 
------------------------------------------

### general

```{r monthlypart-general}
jobsthisay_trim %>% filter(str_detect(Partition,'general') | str_detect(Partition,'test') | str_detect(Partition,'debug')) %>% monthly_summary %>% partpercent(general[1:currmonth]) %>%
  kable() %>% kable_styling(bootstrap_options = c("striped", "hover", "bordered", "condensed", "responsive") )
```

### Serial Jobs

```{r monthlypart-general-serial}
jobsthisay_trim %>% filter(str_detect(Partition,'general') | str_detect(Partition,'test') | str_detect(Partition,'debug')) %>% serial_filter %>% monthly_summary %>% 
  kable() %>% kable_styling(bootstrap_options = c("striped", "hover", "bordered", "condensed", "responsive") )
```

### SMP Jobs

```{r monthly-part-general-smp}
jobsthisay_trim %>% filter(str_detect(Partition,'general') | str_detect(Partition,'test') | str_detect(Partition,'debug')) %>% smp_filter %>% monthly_summary %>%
  kable() %>% kable_styling(bootstrap_options = c("striped", "hover", "bordered", "condensed", "responsive") )
```

### DMP Jobs

```{r monthlypart-general-dmp}
jobsthisay_trim %>% filter(str_detect(Partition,'general') | str_detect(Partition,'test') | str_detect(Partition,'debug')) %>% dmp_filter %>% monthly_summary %>%
  kable() %>% kable_styling(bootstrap_options = c("striped", "hover", "bordered", "condensed", "responsive") )
```

Row 
------------------------------------------

### bigmem

```{r monthlypart-bigmem}
temp <- jobsthisay_trim %>% filter(str_detect(Partition,'general')) %>% monthly_summary %>% select(c("Year","Month"))
jobsthisay_trim %>% filter(str_detect(Partition,'bigmem')) %>% monthly_summary -> month_bigmem
temp1 <- full_join(temp,month_bigmem)
temp1[is.na(temp1)] <- 0
bind_cols(temp1,as.tibble(bigmem[1:currmonth]))  %>%
  mutate(Used=round(SU/value*100,2)) %>%
  select(c(Year,Month,SU,'%Consumed'=Used,Jobs,Wait,Run)) %>%
  kable() %>% kable_styling(bootstrap_options = c("striped", "hover", "bordered", "condensed", "responsive") )
```

### Serial Jobs

```{r monthlypart-bigmem-serial}
jobsthisay_trim %>% filter(str_detect(Partition,'bigmem')) %>% serial_filter %>% monthly_summary %>%
  kable() %>% kable_styling(bootstrap_options = c("striped", "hover", "bordered", "condensed", "responsive") )
```

### SMP Jobs

```{r monthlypart-bigmem-smp}
jobsthisay_trim %>% filter(str_detect(Partition,'bigmem')) %>% smp_filter %>% monthly_summary %>%
  kable() %>% kable_styling(bootstrap_options = c("striped", "hover", "bordered", "condensed", "responsive") )
```

### DMP Jobs

```{r monthlypart-bigmem-dmp}
jobsthisay_trim %>% filter(str_detect(Partition,'bigmem')) %>% dmp_filter %>% monthly_summary %>%
  kable() %>% kable_styling(bootstrap_options = c("striped", "hover", "bordered", "condensed", "responsive") )
```

Row 
------------------------------------------

### gpu

```{r monthlypart-gpu}
jobsthisay_trim %>% filter(str_detect(Partition,'gpu')) %>% monthly_summary %>% partpercent(gpusu[1:currmonth]) %>%
  kable() %>% kable_styling(bootstrap_options = c("striped", "hover", "bordered", "condensed", "responsive") )
```

### Serial Jobs

```{r monthlypart-gpu-serial}
jobsthisay_trim %>% filter(Partition=="gpu") %>% serial_filter %>% monthly_summary %>%
  kable() %>% kable_styling(bootstrap_options = c("striped", "hover", "bordered", "condensed", "responsive") )
```

### SMP Jobs

```{r monthlypart-gpu-smp}
jobsthisay_trim %>% filter(Partition=="gpu") %>% smp_filter %>% monthly_summary %>%
  kable() %>% kable_styling(bootstrap_options = c("striped", "hover", "bordered", "condensed", "responsive") )
```

### DMP Jobs

```{r monthlypart-gpu-dmp}
jobsthisay_trim %>% filter(Partition=="gpu") %>% dmp_filter %>% monthly_summary %>%
  kable() %>% kable_styling(bootstrap_options = c("striped", "hover", "bordered", "condensed", "responsive") )
```

QOS {data-navmenu="Monthly Reports"}
======================================

Row
-------------------------------

### Standard

```{r monthly-qos-summary-std, echo=F}
#listqos <- jobsthisay_trim %>% select(QOS) %>% unique()
#for ( qos in listqos$QOS ) {
  jobsthisay_trim %>% filter(str_detect(QOS,"standard")) %>% monthly_summary %>%
     kable() %>% kable_styling(bootstrap_options = c("striped", "hover", "bordered", "condensed", "responsive") )
#  print("<br />")
#}
```

### Serial Jobs

```{r monthly-qos-summary-std-serial}
jobsthisay_trim %>% filter(str_detect(QOS,"standard")) %>% serial_filter %>% monthly_summary %>%
  kable() %>% kable_styling(bootstrap_options = c("striped", "hover", "bordered", "condensed", "responsive") )
```

### SMP Jobs


```{r monthly-qos-summary-std-smp}
jobsthisay_trim %>% filter(str_detect(QOS,"standard")) %>% smp_filter %>% monthly_summary %>%
  kable() %>% kable_styling(bootstrap_options = c("striped", "hover", "bordered", "condensed", "responsive") )
```

### DMP Jobs


```{r monthly-qos-summary-std-dmp}
jobsthisay_trim %>% filter(str_detect(QOS,"standard")) %>% dmp_filter %>% monthly_summary %>%
  kable() %>% kable_styling(bootstrap_options = c("striped", "hover", "bordered", "condensed", "responsive") )
```

Row
----------------------------
### Low

```{r monthly-qos-summary-low, echo=F}
#listqos <- jobsthisay_trim %>% select(QOS) %>% unique()
#for ( qos in listqos$QOS ) {
  jobsthisay_trim %>% filter(str_detect(QOS,"low")) %>% monthly_summary %>%
     kable() %>% kable_styling(bootstrap_options = c("striped", "hover", "bordered", "condensed", "responsive") )
#  print("<br />")
#}
```

### Serial Jobs

```{r monthly-qos-summary-low-serial}
jobsthisay_trim %>% filter(str_detect(QOS,"low")) %>% serial_filter %>% monthly_summary %>%
  kable() %>% kable_styling(bootstrap_options = c("striped", "hover", "bordered", "condensed", "responsive") )
```

### SMP Jobs


```{r monthly-qos-summary-low-smp}
jobsthisay_trim %>% filter(str_detect(QOS,"low")) %>% smp_filter %>% monthly_summary %>%
  kable() %>% kable_styling(bootstrap_options = c("striped", "hover", "bordered", "condensed", "responsive") )
```

### DMP Jobs


```{r monthly-qos-summary-low-dmp}
jobsthisay_trim %>% filter(str_detect(QOS,"low")) %>% dmp_filter %>% monthly_summary %>%
  kable() %>% kable_styling(bootstrap_options = c("striped", "hover", "bordered", "condensed", "responsive") )
```

Row
----------------------------

### High

```{r monthly-qos-summary-high, echo=F}
#listqos <- jobsthisay_trim %>% select(QOS) %>% unique()
#for ( qos in listqos$QOS ) {
  jobsthisay_trim %>% filter(str_detect(QOS,"high")) %>% monthly_summary %>%
     kable() %>% kable_styling(bootstrap_options = c("striped", "hover", "bordered", "condensed", "responsive") )
#  print("<br />")
#}
```

### Serial Jobs

```{r monthly-qos-summary-high-serial}
jobsthisay_trim %>% filter(str_detect(QOS,"high")) %>% serial_filter %>% monthly_summary %>%
  kable() %>% kable_styling(bootstrap_options = c("striped", "hover", "bordered", "condensed", "responsive") )
```

### SMP Jobs


```{r monthly-qos-summary-high-smp}
jobsthisay_trim %>% filter(str_detect(QOS,"high")) %>% smp_filter %>% monthly_summary %>%
  kable() %>% kable_styling(bootstrap_options = c("striped", "hover", "bordered", "condensed", "responsive") )
```

### DMP Jobs


```{r monthly-qos-summary-high-dmp}
jobsthisay_trim %>% filter(str_detect(QOS,"high")) %>% dmp_filter %>% monthly_summary %>%
  kable() %>% kable_styling(bootstrap_options = c("striped", "hover", "bordered", "condensed", "responsive") )
```





Allocation {data-navmenu="Monthly Reports"}
==========================================

Row {.tabset data-height=1200}
---------------------------------

<!--
### List of PIs

```{r monthlyalloc-list,echo=F}
jobsthisay_trim$Account %>% unique %>% sort %>% matrix(ncol = 1) %>% 
  kable() %>% kable_styling(bootstrap_options = "striped", full_width = F)
```
-->

### Summary

```{r monthlyalloc-summary,echo=F}
jobsthisay_trim %>% 
  mutate(Year=year(End),Month=month(End,label=TRUE)) %>%
  group_by(Year,Month,Account) %>%
  summary %>%
  my_longtable
```

### Partition

```{r monthlyalloc-part,echo=F}
jobsthisay_trim %>% 
  mutate(Year=year(End),Month=month(End,label=TRUE)) %>%
  group_by(Year,Month,Account,Partition) %>%
  summary %>%
  my_longtable
```

### QOS

```{r monthlyalloc-qos,echo=F}
jobsthisay_trim %>% 
  mutate(Year=year(End),Month=month(End,label=TRUE)) %>%
  group_by(Year,Month,Account,QOS) %>%
  summary %>%
  my_longtable
```

User {data-navmenu="Monthly Reports"}
==========================================

Row {.tabset data-height=1200}
---------------------------------

<!--
### List of Users

```{r monthlyuser-list,echo=F}
jobsthisay_trim$User %>% unique %>% sort %>% matrix(ncol = 1) %>% 
  kable() %>% kable_styling(bootstrap_options = "striped")
```
-->

### Summary

```{r monthlyuser-summary,echo=F}
jobsthisay_trim %>% 
  mutate(Year=year(End),Month=month(End,label=TRUE)) %>%
  group_by(Year,Month,User,Account) %>%
  summary %>%
  my_longtable
```

### Partition

```{r monthlyuser-part,echo=F}
jobsthisay_trim %>% 
  mutate(Year=year(End),Month=month(End,label=TRUE)) %>%
  group_by(Year,Month,User,Account,Partition) %>%
  summary %>%
  my_longtable
```

### QOS

```{r monthlyuser-qos,echo=F}
jobsthisay_trim %>% 
  mutate(Year=year(End),Month=month(End,label=TRUE)) %>%
  group_by(Year,Month,User,Account,QOS) %>%
  summary %>%
  my_longtable
```