output$my_table2 <- DT::renderDataTable({
    datatable(iris[1:100, c(1:5)],
            rownames = FALSE, 
            extensions = 'RowGroup', 
            options = list(rowGroup = list(dataSrc=c(4))
            )
  )})

# compute subtotals of 'mpg' by level of 'cyl'
byFactor <- "cyl"
variable <- "mpg"
subtotals <- sapply(split(mtcars, mtcars[[byFactor]]), function(dat) {
  sum(dat[[variable]])
})
n <- length(subtotals)
# move the rownames of 'mtcars' to first column
dat0 <- mtcars
dat0 <- cbind(car = rownames(mtcars), mtcars)
# append the subtotals to this dataframe 'dat0'
totals <- matrix(NA, nrow = n, ncol = ncol(dat0))
colnames(totals) <- colnames(dat0)
dat <- rbind(dat0, totals)
dat_tail <- (nrow(dat)-n+1L):nrow(dat) # the row indices of the new data
dat[dat_tail, "car"] <- "Total"
dat[dat_tail, variable] <- subtotals
dat[dat_tail, byFactor] <- names(subtotals)
# sort 'dat' by the 'cyl' column
dat <- dat[order(dat[[byFactor]]), ]
# now make the datatable
library(DT)
datatable(
  dat, rownames = FALSE, 
  extensions = "RowGroup", 
  options = list(
    rowGroup = list(dataSrc = list(2))
  )
)

# extract indices of the rows to be bolded
indices <- which(dat[["car"]] == "Total") - 1L # subtract 1 for JavaScript!
cb <- JS(
  "function(row, data, displayNum, index) {", 
  sprintf("var indices = [%s];", toString(indices)),
  "  if(indices.indexOf(index) > -1) {",
  "    $(row).css('font-weight', 'bold');",
  "  }",
  "}"
)
library(DT)
datatable(
  dat, rownames = FALSE, 
  extensions = "RowGroup", 
  options = list(
    rowCallback = cb,
    rowGroup = list(dataSrc = list(2))
  )
)

# I would also prepend 'cyl:' to the rowgroup headers and hide the 'cyl' column
dat[[byFactor]] <- paste0(byFactor, ": ", dat[[byFactor]])
datatable(
  dat, rownames = FALSE, 
  extensions = "RowGroup", 
  options = list(
    rowCallback = cb,
    rowGroup = list(dataSrc = list(2)),
    columnDefs = list(
      list(targets = 2, visible = FALSE)
    )
  )
)