Copyright	(c) Colin Woodbury 2015 - 2018
License	GPL3
Maintainer	Colin Woodbury <colingw@gmail.com>
Safe Haskell	None
Language	Haskell2010

Data.Kanji

Contents

Kanji
Character Categories
Levels
Analysis
- Densities

Description

A library for analysing the density of Kanji in given texts, according to their Level classification, as defined by the Japan Kanji Aptitude Testing Foundation (日本漢字能力検定協会).

Synopsis

data Kanji
kanji :: Char -> Maybe Kanji
_kanji :: Kanji -> Char
allKanji :: Map Level (Set Kanji)
isKanji :: Char -> Bool
isHiragana :: Char -> Bool
isKatakana :: Char -> Bool
data CharCat
- = Hanzi
- | Hiragana
- | Katakana
- | Numeral
- | RomanLetter
- | Punctuation
- | Other
category :: Char -> CharCat
data Level
- = Ten
- | Nine
- | Eight
- | Seven
- | Six
- | Five
- | Four
- | Three
- | PreTwo
- | Two
- | PreOne
- | One
- | Unknown
level :: Kanji -> Level
percentSpread :: [Kanji] -> Map Kanji Float
levelDist :: [Kanji] -> Map Level Float
uniques :: [Kanji] -> Map Level (Set Kanji)
densities :: Text -> Map CharCat Float
elementaryDen :: Map Level Float -> Float
middleDen :: Map Level Float -> Float
highDen :: Map Level Float -> Float

Kanji

data Kanji #

A single symbol of Kanji. Japanese Kanji were borrowed from China over several waves during the last 1,500 years. Japan names 2,136 of these as their standard set, with rarer characters being the domain of academia and esoteric writers.

Japanese has several Japan-only Kanji, including:

畑 (a type of rice field)
峠 (a narrow mountain pass)
働 (to do physical labour)

Instances

Eq Kanji #
Instance details Defined in Data.Kanji.Types Methods (==) :: Kanji -> Kanji -> Bool # (/=) :: Kanji -> Kanji -> Bool #
Ord Kanji #
Instance details Defined in Data.Kanji.Types Methods compare :: Kanji -> Kanji -> Ordering # (<) :: Kanji -> Kanji -> Bool # (<=) :: Kanji -> Kanji -> Bool # (>) :: Kanji -> Kanji -> Bool # (>=) :: Kanji -> Kanji -> Bool # max :: Kanji -> Kanji -> Kanji # min :: Kanji -> Kanji -> Kanji #
Show Kanji #
Instance details Defined in Data.Kanji.Types Methods showsPrec :: Int -> Kanji -> ShowS # show :: Kanji -> String # showList :: [Kanji] -> ShowS #
Generic Kanji #
Instance details Defined in Data.Kanji.Types Associated Types type Rep Kanji :: * -> * # Methods from :: Kanji -> Rep Kanji x # to :: Rep Kanji x -> Kanji #
Hashable Kanji #
Instance details Defined in Data.Kanji.Types Methods hashWithSalt :: Int -> Kanji -> Int # hash :: Kanji -> Int #
ToJSON Kanji #
Instance details Defined in Data.Kanji.Types Methods toJSON :: Kanji -> Value # toEncoding :: Kanji -> Encoding # toJSONList :: [Kanji] -> Value # toEncodingList :: [Kanji] -> Encoding #
FromJSON Kanji #
Instance details Defined in Data.Kanji.Types Methods parseJSON :: Value -> Parser Kanji # parseJSONList :: Value -> Parser [Kanji] #
NFData Kanji #
Instance details Defined in Data.Kanji.Types Methods rnf :: Kanji -> () #
type Rep Kanji #
Instance details Defined in Data.Kanji.Types type Rep Kanji = D1 (MetaData "Kanji" "Data.Kanji.Types" "kanji-3.4.0-DVGCwVO8stlDIxqk0jYmH5" True) (C1 (MetaCons "Kanji" PrefixI False) (S1 (MetaSel (Nothing :: Maybe Symbol) NoSourceUnpackedness NoSourceStrictness DecidedLazy) (Rec0 Char)))

kanji :: Char -> Maybe Kanji #

Construct a Kanji value from some Char if it falls in the correct UTF8 range.

_kanji :: Kanji -> Char #

The original Char of a Kanji.

allKanji :: Map Level (Set Kanji) #

All Japanese Kanji, grouped by their Level (級).

isKanji :: Char -> Bool #

Legal Kanji appear between UTF-8 characters 19968 and 40959.

isHiragana :: Char -> Bool #

あ to ん.

isKatakana :: Char -> Bool #

ア to ン.

Character Categories

data CharCat #

General categories for characters, at least as is useful for thinking about Japanese.

Japanese "full-width" numbers and letters will be counted as Numeral and RomanLetter respectively, alongside their usual ASCII forms.

Constructors

Hanzi
Hiragana
Katakana
Numeral
RomanLetter
Punctuation
Other

Instances

Eq CharCat #
Instance details Defined in Data.Kanji.Types Methods (==) :: CharCat -> CharCat -> Bool # (/=) :: CharCat -> CharCat -> Bool #
Ord CharCat #
Instance details Defined in Data.Kanji.Types Methods compare :: CharCat -> CharCat -> Ordering # (<) :: CharCat -> CharCat -> Bool # (<=) :: CharCat -> CharCat -> Bool # (>) :: CharCat -> CharCat -> Bool # (>=) :: CharCat -> CharCat -> Bool # max :: CharCat -> CharCat -> CharCat # min :: CharCat -> CharCat -> CharCat #
Show CharCat #
Instance details Defined in Data.Kanji.Types Methods showsPrec :: Int -> CharCat -> ShowS # show :: CharCat -> String # showList :: [CharCat] -> ShowS #
Generic CharCat #
Instance details Defined in Data.Kanji.Types Associated Types type Rep CharCat :: * -> * # Methods from :: CharCat -> Rep CharCat x # to :: Rep CharCat x -> CharCat #
Hashable CharCat #
Instance details Defined in Data.Kanji.Types Methods hashWithSalt :: Int -> CharCat -> Int # hash :: CharCat -> Int #
ToJSON CharCat #
Instance details Defined in Data.Kanji.Types Methods toJSON :: CharCat -> Value # toEncoding :: CharCat -> Encoding # toJSONList :: [CharCat] -> Value # toEncodingList :: [CharCat] -> Encoding #
ToJSONKey CharCat #
Instance details Defined in Data.Kanji.Types Methods toJSONKey :: ToJSONKeyFunction CharCat # toJSONKeyList :: ToJSONKeyFunction [CharCat] #
FromJSON CharCat #
Instance details Defined in Data.Kanji.Types Methods parseJSON :: Value -> Parser CharCat # parseJSONList :: Value -> Parser [CharCat] #
NFData CharCat #
Instance details Defined in Data.Kanji.Types Methods rnf :: CharCat -> () #
type Rep CharCat #
Instance details Defined in Data.Kanji.Types type Rep CharCat = D1 (MetaData "CharCat" "Data.Kanji.Types" "kanji-3.4.0-DVGCwVO8stlDIxqk0jYmH5" False) ((C1 (MetaCons "Hanzi" PrefixI False) (U1 :: * -> ) :+: (C1 (MetaCons "Hiragana" PrefixI False) (U1 :: -> ) :+: C1 (MetaCons "Katakana" PrefixI False) (U1 :: -> ))) :+: ((C1 (MetaCons "Numeral" PrefixI False) (U1 :: -> ) :+: C1 (MetaCons "RomanLetter" PrefixI False) (U1 :: -> )) :+: (C1 (MetaCons "Punctuation" PrefixI False) (U1 :: -> ) :+: C1 (MetaCons "Other" PrefixI False) (U1 :: -> *))))

category :: Char -> CharCat #

Levels

data Level #

A Level or Kyuu (級) of Japanese Kanji ranking. There are 12 of these, from 10 to 1, including intermediate levels between 3 and 2, and 2 and 1.

Japanese students will typically have Level-5 ability by the time they finish elementary school. Level-5 accounts for 1,006 characters.

By the end of middle school, they would have covered up to Level-3 (1607 Kanji) in their Japanese class curriculum.

While Level-2 (2,136 Kanji) is considered "standard adult" ability, many adults could not pass the Level-2, or even the Level-Pre2 (1940 Kanji) exam without considerable study.

Level data for Kanji above Level-2 is currently not provided by this library.

Constructors

Ten
Nine
Eight
Seven
Six
Five
Four
Three
PreTwo
Two
PreOne
One
Unknown

Instances

Enum Level #
Instance details Defined in Data.Kanji.Types Methods succ :: Level -> Level # pred :: Level -> Level # toEnum :: Int -> Level # fromEnum :: Level -> Int # enumFrom :: Level -> [Level] # enumFromThen :: Level -> Level -> [Level] # enumFromTo :: Level -> Level -> [Level] # enumFromThenTo :: Level -> Level -> Level -> [Level] #
Eq Level #
Instance details Defined in Data.Kanji.Types Methods (==) :: Level -> Level -> Bool # (/=) :: Level -> Level -> Bool #
Ord Level #
Instance details Defined in Data.Kanji.Types Methods compare :: Level -> Level -> Ordering # (<) :: Level -> Level -> Bool # (<=) :: Level -> Level -> Bool # (>) :: Level -> Level -> Bool # (>=) :: Level -> Level -> Bool # max :: Level -> Level -> Level # min :: Level -> Level -> Level #
Show Level #
Instance details Defined in Data.Kanji.Types Methods showsPrec :: Int -> Level -> ShowS # show :: Level -> String # showList :: [Level] -> ShowS #
Generic Level #
Instance details Defined in Data.Kanji.Types Associated Types type Rep Level :: * -> * # Methods from :: Level -> Rep Level x # to :: Rep Level x -> Level #
Hashable Level #
Instance details Defined in Data.Kanji.Types Methods hashWithSalt :: Int -> Level -> Int # hash :: Level -> Int #
ToJSON Level #
Instance details Defined in Data.Kanji.Types Methods toJSON :: Level -> Value # toEncoding :: Level -> Encoding # toJSONList :: [Level] -> Value # toEncodingList :: [Level] -> Encoding #
ToJSONKey Level #
Instance details Defined in Data.Kanji.Types Methods toJSONKey :: ToJSONKeyFunction Level # toJSONKeyList :: ToJSONKeyFunction [Level] #
FromJSON Level #
Instance details Defined in Data.Kanji.Types Methods parseJSON :: Value -> Parser Level # parseJSONList :: Value -> Parser [Level] #
NFData Level #
Instance details Defined in Data.Kanji.Types Methods rnf :: Level -> () #
type Rep Level #
Instance details Defined in Data.Kanji.Types type Rep Level = D1 (MetaData "Level" "Data.Kanji.Types" "kanji-3.4.0-DVGCwVO8stlDIxqk0jYmH5" False) (((C1 (MetaCons "Ten" PrefixI False) (U1 :: * -> ) :+: (C1 (MetaCons "Nine" PrefixI False) (U1 :: -> ) :+: C1 (MetaCons "Eight" PrefixI False) (U1 :: -> ))) :+: (C1 (MetaCons "Seven" PrefixI False) (U1 :: -> ) :+: (C1 (MetaCons "Six" PrefixI False) (U1 :: -> ) :+: C1 (MetaCons "Five" PrefixI False) (U1 :: -> )))) :+: ((C1 (MetaCons "Four" PrefixI False) (U1 :: -> ) :+: (C1 (MetaCons "Three" PrefixI False) (U1 :: -> ) :+: C1 (MetaCons "PreTwo" PrefixI False) (U1 :: -> ))) :+: ((C1 (MetaCons "Two" PrefixI False) (U1 :: -> ) :+: C1 (MetaCons "PreOne" PrefixI False) (U1 :: -> )) :+: (C1 (MetaCons "One" PrefixI False) (U1 :: -> ) :+: C1 (MetaCons "Unknown" PrefixI False) (U1 :: -> *)))))

level :: Kanji -> Level #

What Level does a Kanji belong to? Unknown for Kanji above level Two.

Analysis

percentSpread :: [Kanji] -> Map Kanji Float #

The distribution of each Kanji in a set of them. The distribution values must sum to 1.

levelDist :: [Kanji] -> Map Level Float #

How much of each Level is represented by a group of Kanji? The distribution values will sum to 1.

uniques :: [Kanji] -> Map Level (Set Kanji) #

Which Kanji appeared from each Level in the text?

Densities

densities :: Text -> Map CharCat Float #

Percentage of appearance of each CharCat in the source text. The percentages will sum to 1.0.

elementaryDen :: Map Level Float -> Float #

How much of the Kanji found are learnt in elementary school in Japan?

elementaryDen . levelDist :: [Kanji] -> Float

middleDen :: Map Level Float -> Float #

How much of the Kanji found are learnt by the end of middle school?

middleDen . levelDist :: [Kanji] -> Float

highDen :: Map Level Float -> Float #

How much of the Kanji found are learnt by the end of high school?

highDen . levelDist :: [Kanji] -> Float