Change internal representation of dataframe to not have "holes" to be filled.

Since the data structure is iimmutable this is largely pointless.

Author: mchav

benchmark/Main.hs

@@ -10,21 +10,19 @@ import System.Random (randomRIO)
 
 stats :: Int -> IO ()
 stats n = do
-  ns <- VU.replicateM n (randomRIO (-20.0 :: Double, 20.0))
-  xs <- VU.replicateM n (randomRIO (-20.0 :: Double, 20.0))
-  ys <- VU.replicateM n (randomRIO (-20.0 :: Double, 20.0))
-  let df = D.fromNamedColumns [("first", D.UnboxedColumn ns),
-                               ("second", D.UnboxedColumn xs),
-                               ("third", D.UnboxedColumn ys)]
+  ns <- do
+    ns' <- VU.replicateM n (randomRIO (-20.0 :: Double, 20.0))
+    pure $ replicate 3 ns'
+  let df = D.fromUnamedColumns (map D.fromUnboxedVector ns)
 
-  print $ D.mean "first" df
-  print $ D.variance "second" df
-  print $ D.correlation "second" "third" df
-  print $ D.select ["first"] df D.|> D.take 1
+  print $ D.mean "0" df
+  print $ D.variance "1" df
+  print $ D.correlation "1" "2" df
+  print $ D.filter "0" (>= (19.9 :: Double)) df D.|> D.take 10
 
 main = defaultMain [
-  bgroup "stats" [ bench    "300_000" $ nfIO (stats 100_000)
+  bgroup "stats" [ bench    "300_000" $ toBenchmarkable  (stats 100_000)
                  , bench  "3_000_000" $ nfIO (stats 1_000_000)
-                 , bench "30_000_000" $ nfIO (stats 30_000_000)
+                 , bench "300_000_000" $ nfIO (stats 300_000_000)
                  ]
   ]

dataframe.cabal

@@ -22,6 +22,7 @@ source-repository head
   location: https://github.com/mchav/dataframe
 
 library
+    default-extensions: StrictData
     exposed-modules: DataFrame,
                      DataFrame.Lazy
     other-modules: DataFrame.Internal.Types,

src/DataFrame/Display/Terminal/Plot.hs

@@ -55,33 +55,32 @@ plotHistogramsBy col plotSet orientation df = do
         plotForColumnBy col cname byColumn plotColumn orientation df
 
 -- Plot code adapted from: https://alexwlchan.net/2018/ascii-bar-charts/
-plotForColumnBy :: HasCallStack => T.Text -> T.Text -> Maybe Column -> Maybe Column -> HistogramOrientation -> DataFrame -> IO ()
-plotForColumnBy _ _ Nothing _ _ _ = return ()
-plotForColumnBy byCol cname (Just (BoxedColumn (byColumn :: V.Vector a))) (Just (BoxedColumn (plotColumn :: V.Vector b))) orientation df = do
+plotForColumnBy :: HasCallStack => T.Text -> T.Text -> Column -> Column -> HistogramOrientation -> DataFrame -> IO ()
+plotForColumnBy byCol cname (BoxedColumn (byColumn :: V.Vector a)) (BoxedColumn (plotColumn :: V.Vector b)) orientation df = do
     let zipped = VG.zipWith (\left right -> (show left, show right)) plotColumn byColumn
     let counts = countOccurrences zipped
     if null counts || length counts > 20
     then pure ()
     else case orientation of
         VerticalHistogram -> error "Vertical histograms aren't yet supported"
         HorizontalHistogram -> plotGivenCounts' cname counts
-plotForColumnBy byCol cname (Just (UnboxedColumn byColumn)) (Just (BoxedColumn plotColumn)) orientation df = do
+plotForColumnBy byCol cname (UnboxedColumn byColumn) (BoxedColumn plotColumn) orientation df = do
     let zipped = VG.zipWith (\left right -> (show left, show right)) plotColumn (V.convert byColumn)
     let counts = countOccurrences zipped
     if null counts || length counts > 20
     then pure ()
     else case orientation of
         VerticalHistogram -> error "Vertical histograms aren't yet supported"
         HorizontalHistogram -> plotGivenCounts' cname counts
-plotForColumnBy byCol cname (Just (BoxedColumn byColumn)) (Just (UnboxedColumn plotColumn)) orientation df = do
+plotForColumnBy byCol cname (BoxedColumn byColumn) (UnboxedColumn plotColumn) orientation df = do
     let zipped = VG.zipWith (\left right -> (show left, show right)) (V.convert plotColumn) (V.convert byColumn)
     let counts = countOccurrences zipped
     if null counts || length counts > 20
     then pure ()
     else case orientation of
         -- VerticalHistogram -> plotVerticalGivenCounts cname counts
         HorizontalHistogram -> plotGivenCounts' cname counts
-plotForColumnBy byCol cname (Just (UnboxedColumn byColumn)) (Just (UnboxedColumn plotColumn)) orientation df = do
+plotForColumnBy byCol cname (UnboxedColumn byColumn) (UnboxedColumn plotColumn) orientation df = do
     let zipped = VG.zipWith (\left right -> (show left, show right)) (V.convert plotColumn) (V.convert byColumn)
     let counts = countOccurrences zipped
     if null counts || length counts > 20
@@ -93,9 +92,8 @@ plotForColumnBy byCol cname (Just (UnboxedColumn byColumn)) (Just (UnboxedColumn
 plotForColumnBy _ _ _ _ _ _ = return ()
 
 -- Plot code adapted from: https://alexwlchan.net/2018/ascii-bar-charts/
-plotForColumn :: HasCallStack => T.Text -> Maybe Column -> HistogramOrientation -> DataFrame -> IO ()
-plotForColumn _ Nothing _ _ = return ()
-plotForColumn cname (Just (BoxedColumn (column :: V.Vector a))) orientation df = do
+plotForColumn :: HasCallStack => T.Text -> Column -> HistogramOrientation -> DataFrame -> IO ()
+plotForColumn cname (BoxedColumn (column :: V.Vector a)) orientation df = do
     let repa :: Ref.TypeRep a = Ref.typeRep @a
         repText :: Ref.TypeRep T.Text = Ref.typeRep @T.Text
         repString :: Ref.TypeRep String = Ref.typeRep @String
@@ -110,7 +108,7 @@ plotForColumn cname (Just (BoxedColumn (column :: V.Vector a))) orientation df =
     else case orientation of
         VerticalHistogram -> plotVerticalGivenCounts cname counts
         HorizontalHistogram -> plotGivenCounts cname counts
-plotForColumn cname (Just (UnboxedColumn (column :: VU.Vector a))) orientation df = do
+plotForColumn cname (UnboxedColumn (column :: VU.Vector a)) orientation df = do
     let repa :: Ref.TypeRep a = Ref.typeRep @a
         repText :: Ref.TypeRep T.Text = Ref.typeRep @T.Text
         repString :: Ref.TypeRep String = Ref.typeRep @String

src/DataFrame/IO/CSV.hs

@@ -1,12 +1,10 @@
-{-# LANGUAGE BangPatterns #-}
 {-# LANGUAGE ExplicitNamespaces #-}
 {-# LANGUAGE LambdaCase #-}
 {-# LANGUAGE OverloadedStrings #-}
 {-# LANGUAGE ScopedTypeVariables #-}
 {-# LANGUAGE TypeApplications #-}
 {-# LANGUAGE GADTs #-}
 {-# LANGUAGE RankNTypes #-}
-{-# LANGUAGE Strict #-}
 module DataFrame.IO.CSV where
 
 import qualified Data.ByteString.Char8 as C
@@ -104,9 +102,8 @@ readSeparated c opts path = do
         cols <- V.mapM (freezeColumn mutableCols nulls' opts) (V.generate numColumns id)
         return $ DataFrame {
                 columns = cols,
-                freeIndices = [],
                 columnIndices = M.fromList (zip columnNames [0..]),
-                dataframeDimensions = (maybe 0 columnLength (cols V.! 0), V.length cols)
+                dataframeDimensions = (maybe 0 columnLength (cols V.!? 0), V.length cols)
             }
 {-# INLINE readSeparated #-}
 
@@ -160,10 +157,10 @@ writeValue mutableCols nullIndices count colIndex value = do
 {-# INLINE writeValue #-}
 
 -- | Freezes a mutable vector into an immutable one, trimming it to the actual row count.
-freezeColumn :: VM.IOVector Column -> V.Vector [(Int, T.Text)] -> ReadOptions -> Int -> IO (Maybe Column)
+freezeColumn :: VM.IOVector Column -> V.Vector [(Int, T.Text)] -> ReadOptions -> Int -> IO Column
 freezeColumn mutableCols nulls opts colIndex = do
     col <- VM.unsafeRead mutableCols colIndex
-    Just <$> freezeColumn' (nulls V.! colIndex) col
+    freezeColumn' (nulls V.! colIndex) col
 {-# INLINE freezeColumn #-}
 
 parseSep :: Char -> T.Text -> [T.Text]
@@ -215,7 +212,7 @@ lineEnd =
 countRows :: Char -> FilePath -> IO Int
 countRows c path = withFile path ReadMode $! go 0 ""
    where
-      go !n !input h = do
+      go n input h = do
          isEOF <- hIsEOF h
          if isEOF && input == mempty
             then pure n
@@ -250,8 +247,7 @@ getRowAsText :: DataFrame -> Int -> [T.Text]
 getRowAsText df i = V.ifoldr go [] (columns df)
   where
     indexMap = M.fromList (map (\(a, b) -> (b, a)) $ M.toList (columnIndices df))
-    go k Nothing acc = acc
-    go k (Just (BoxedColumn (c :: V.Vector a))) acc = case c V.!? i of
+    go k (BoxedColumn (c :: V.Vector a)) acc = case c V.!? i of
         Just e -> textRep : acc
             where textRep = case testEquality (typeRep @a) (typeRep @T.Text) of
                     Just Refl -> e
@@ -272,7 +268,7 @@ getRowAsText df i = V.ifoldr go [] (columns df)
                 ++ " has less items than "
                 ++ "the other columns at index "
                 ++ show i
-    go k (Just (UnboxedColumn c)) acc = case c VU.!? i of
+    go k (UnboxedColumn c) acc = case c VU.!? i of
         Just e -> T.pack (show e) : acc
         Nothing ->
             error $
@@ -281,7 +277,7 @@ getRowAsText df i = V.ifoldr go [] (columns df)
                 ++ " has less items than "
                 ++ "the other columns at index "
                 ++ show i
-    go k (Just (OptionalColumn (c :: V.Vector (Maybe a)))) acc = case c V.!? i of
+    go k (OptionalColumn (c :: V.Vector (Maybe a))) acc = case c V.!? i of
         Just e -> textRep : acc
             where textRep = case testEquality (typeRep @a) (typeRep @T.Text) of
                     Just Refl -> fromMaybe "Nothing" e

src/DataFrame/Internal/Column.hs

@@ -4,7 +4,6 @@
 {-# LANGUAGE OverloadedStrings #-}
 {-# LANGUAGE RankNTypes #-}
 {-# LANGUAGE ScopedTypeVariables #-}
-{-# LANGUAGE Strict #-}
 {-# LANGUAGE TypeApplications #-}
 {-# LANGUAGE FlexibleContexts #-}
 {-# LANGUAGE FlexibleInstances #-}
@@ -254,17 +253,19 @@ fromList ::
   => [a] -> Column
 fromList = toColumnRep @(KindOf a) . VB.fromList
 
-
+-- | Type-level boolean for constraint/type comparison.
 data SBool (b :: Bool) where
   STrue  :: SBool 'True
   SFalse :: SBool 'False
 
+-- | The runtime witness for our type-level branching.
 class SBoolI (b :: Bool) where
-  sbool :: SBool b          -- the run-time witness
+  sbool :: SBool b
 
 instance SBoolI 'True  where sbool = STrue
 instance SBoolI 'False where sbool = SFalse
 
+-- | Type-level function to determine whether or not a type is unboxa
 sUnbox :: forall a. SBoolI (Unboxable a) => SBool (Unboxable a)
 sUnbox = sbool @(Unboxable a)
 
@@ -564,27 +565,37 @@ freezeColumn' nulls (MutableUnboxedColumn col)
 
 -- | Fills the end of a column, up to n, with Nothing. Does nothing if column has length greater than n.
 expandColumn :: Int -> Column -> Column
-expandColumn n (OptionalColumn col) = OptionalColumn $ col <> VB.replicate n Nothing
+expandColumn n (OptionalColumn col) = OptionalColumn $ col <> VB.replicate (n - VG.length col) Nothing
 expandColumn n column@(BoxedColumn col)
-  | n < VG.length col = OptionalColumn $ VB.map Just col <> VB.replicate n Nothing
+  | n > VG.length col = OptionalColumn $ VB.map Just col <> VB.replicate (n - VG.length col) Nothing
   | otherwise         = column
 expandColumn n column@(UnboxedColumn col)
-  | n < VG.length col = OptionalColumn $ VB.map Just (VU.convert col) <> VB.replicate n Nothing
+  | n > VG.length col = OptionalColumn $ VB.map Just (VU.convert col) <> VB.replicate (n - VG.length col) Nothing
   | otherwise         = column
 expandColumn n column@(GroupedBoxedColumn col)
-  | n < VG.length col = GroupedBoxedColumn $ col <> VB.replicate n VB.empty
+  | n > VG.length col = GroupedBoxedColumn $ col <> VB.replicate (n - VG.length col) VB.empty
   | otherwise         = column
 expandColumn n column@(GroupedUnboxedColumn col)
-  | n < VG.length col = GroupedUnboxedColumn $ col <> VB.replicate n VU.empty
+  | n > VG.length col = GroupedUnboxedColumn $ col <> VB.replicate (n - VG.length col) VU.empty
   | otherwise         = column
 
 -- | Fills the beginning of a column, up to n, with Nothing. Does nothing if column has length greater than n.
 leftExpandColumn :: Int -> Column -> Column
-leftExpandColumn n (OptionalColumn col) = OptionalColumn $ VB.replicate n Nothing <> col
-leftExpandColumn n (BoxedColumn col) = OptionalColumn $ VB.replicate n Nothing <> VB.map Just col
-leftExpandColumn n (UnboxedColumn col) = OptionalColumn $ VB.replicate n Nothing <> VB.map Just (VU.convert col)
-leftExpandColumn n (GroupedBoxedColumn col) = GroupedBoxedColumn $ VB.replicate n VB.empty <> col
-leftExpandColumn n (GroupedUnboxedColumn col) = GroupedUnboxedColumn $ VB.replicate n VU.empty <> col
+leftExpandColumn n column@(OptionalColumn col)
+  | n > VG.length col = OptionalColumn $ VG.replicate (n - VG.length col) Nothing <> col
+  | otherwise         = column
+leftExpandColumn n column@(BoxedColumn col)
+  | n > VG.length col = OptionalColumn $ VG.replicate (n - VG.length col) Nothing <> VG.map Just col
+  | otherwise         = column
+leftExpandColumn n column@(UnboxedColumn col)
+  | n > VG.length col = OptionalColumn $ VG.replicate (n - VG.length col) Nothing <> VG.map Just (VU.convert col)
+  | otherwise         = column
+leftExpandColumn n column@(GroupedBoxedColumn col)
+  | n > VG.length col = GroupedBoxedColumn $ VG.replicate (n - VG.length col) VB.empty <> col
+  | otherwise         = column
+leftExpandColumn n column@(GroupedUnboxedColumn col)
+  | n > VG.length col = GroupedUnboxedColumn $ VG.replicate (n - VG.length col) VU.empty <> col
+  | otherwise         = column
 
 -- | Concatenates two columns.
 concatColumns :: Column -> Column -> Maybe Column

src/DataFrame/Internal/DataFrame.hs

@@ -4,7 +4,6 @@
 {-# LANGUAGE ScopedTypeVariables #-}
 {-# LANGUAGE TypeApplications #-}
 {-# LANGUAGE GADTs #-}
-{-# LANGUAGE Strict #-}
 {-# LANGUAGE FlexibleContexts #-}
 module DataFrame.Internal.DataFrame where
 
@@ -25,11 +24,9 @@ import Type.Reflection (typeRep)
 data DataFrame = DataFrame
   { -- | Our main data structure stores a dataframe as
     -- a vector of columns. This improv
-    columns :: V.Vector (Maybe Column),
+    columns :: V.Vector Column,
     -- | Keeps the column names in the order they were inserted in.
     columnIndices :: M.Map T.Text Int,
-    -- | Next free index that we insert a column into.
-    freeIndices :: [Int],
     dataframeDimensions :: (Int, Int)
   }
 
@@ -46,13 +43,12 @@ asText :: DataFrame -> Bool -> T.Text
 asText d properMarkdown =
   let header = "index" : map fst (sortBy (compare `on` snd) $ M.toList (columnIndices d))
       types = V.toList $ V.filter (/= "") $ V.map getType (columns d)
-      getType :: Maybe Column -> T.Text
-      getType Nothing = ""
-      getType (Just (BoxedColumn (column :: V.Vector a))) = T.pack $ show (typeRep @a)
-      getType (Just (UnboxedColumn (column :: VU.Vector a))) = T.pack $ show (typeRep @a)
-      getType (Just (OptionalColumn (column :: V.Vector a))) = T.pack $ show (typeRep @a)
-      getType (Just (GroupedBoxedColumn (column :: V.Vector a))) = T.pack $ show (typeRep @a)
-      getType (Just (GroupedUnboxedColumn (column :: V.Vector a))) = T.pack $ show (typeRep @a)
+      getType :: Column -> T.Text
+      getType (BoxedColumn (column :: V.Vector a)) = T.pack $ show (typeRep @a)
+      getType (UnboxedColumn (column :: VU.Vector a)) = T.pack $ show (typeRep @a)
+      getType (OptionalColumn (column :: V.Vector a)) = T.pack $ show (typeRep @a)
+      getType (GroupedBoxedColumn (column :: V.Vector a)) = T.pack $ show (typeRep @a)
+      getType (GroupedUnboxedColumn (column :: V.Vector a)) = T.pack $ show (typeRep @a)
       -- Separate out cases dynamically so we don't end up making round trip string
       -- copies.
       get :: Maybe Column -> V.Vector T.Text
@@ -67,32 +63,22 @@ asText d properMarkdown =
       get (Just (GroupedUnboxedColumn column)) = V.map (T.pack . show) column
       getTextColumnFromFrame df (i, name) = if i == 0
                                             then V.fromList (map (T.pack . show) [0..(fst (dataframeDimensions df) - 1)])
-                                            else get $ (V.!) (columns d) ((M.!) (columnIndices d) name)
+                                            else get $ (V.!?) (columns d) ((M.!) (columnIndices d) name)
       rows =
         transpose $
           zipWith (curry (V.toList . getTextColumnFromFrame d)) [0..] header
    in (if properMarkdown then showTableProperMarkdown else showTable) header ("Int":types) rows
 
 -- | O(1) Creates an empty dataframe
 empty :: DataFrame
-empty = DataFrame {columns = V.replicate initialColumnSize Nothing,
+empty = DataFrame {columns = V.empty,
                    columnIndices = M.empty,
-                   freeIndices = [0..(initialColumnSize - 1)],
                    dataframeDimensions = (0, 0) }
 
-initialColumnSize :: Int
-initialColumnSize = 8
-
 getColumn :: T.Text -> DataFrame -> Maybe Column
 getColumn name df = do
   i <- columnIndices df M.!? name
-  join $ columns df V.!? i
+  columns df V.!? i
 
 null :: DataFrame -> Bool
-null df = dataframeDimensions df == (0, 0)
-
-metadata :: DataFrame -> String
-metadata df = show (columnIndices df) ++ "\n" ++
-              show (V.map (fmap columnVersionString) (columns df)) ++ "\n" ++
-              show (freeIndices df) ++ "\n" ++
-              show (dataframeDimensions df)
+null df = V.null (columns df)

src/DataFrame/Internal/Expression.hs

@@ -4,7 +4,6 @@
 {-# LANGUAGE OverloadedStrings #-}
 {-# LANGUAGE RankNTypes #-}
 {-# LANGUAGE ScopedTypeVariables #-}
-{-# LANGUAGE StrictData #-}
 {-# LANGUAGE TypeApplications #-}
 {-# LANGUAGE FlexibleContexts #-}
 {-# LANGUAGE FlexibleInstances #-}

src/DataFrame/Internal/Parsing.hs

@@ -1,5 +1,4 @@
 {-# LANGUAGE OverloadedStrings #-}
-{-# LANGUAGE Strict #-}
 module DataFrame.Internal.Parsing where
 
 import qualified Data.ByteString.Char8 as C

src/DataFrame/Internal/Types.hs

@@ -8,7 +8,6 @@
 {-# LANGUAGE ScopedTypeVariables #-}
 {-# LANGUAGE TypeApplications #-}
 {-# LANGUAGE TypeOperators #-}
-{-# LANGUAGE Strict #-}
 module DataFrame.Internal.Types where
 
 import Data.Int ( Int8, Int16, Int32, Int64 )