I'm please to announce the NetBeans (6.8) plugin for Scala 2.8.0 RC1 is available now. It's a maintain release to catch up with Scala 2.8.0 RC1.

The packed plugins is downloadable at:  http://sourceforge.net/projects/erlybird/files/nb-scala/6.8v1.1.0rc2/

It requires Scala 2.8.0 RC1, which is available at:  http://www.scala-lang.org/downloads. For maven project, it's under  http://www.scala-tools.org/repo-releases, with version number "2.8.0.RC1"

Please refer to  http://wiki.netbeans.org/Scala68v1 for the installation/upgrading information.

About one and half year ago, it was my first time to consider Scala seriously, I wrote a blog about the syntax example of Scala, Erlang and Haskell .

With more experience of Scala, I'd like to know how about the functional ability of Scala comparing to Haskell. I picked up  Paul R. Brown's perpubplat blog engine again, which is a Haskell implementation heavily in functional style. I tried to port more code from Haskell to Scala keeping in similar expressions. Here's the code example of Entry.scala in Scala comparing to Brown's original Entry.hs:

Original Haskell code piece

-- | Data structures for an item (post or comment) and the
-- overall structure in terms of parents and children.
module Blog.Model.Entry where

import qualified Blog.FrontEnd.Urls as U
import Utilities
import qualified Blog.Constants as C

import Maybe
import List ( sortBy, isPrefixOf, intersperse)
import qualified Data.Map as M
import Data.Map ( (!) )

type ISO8601DatetimeString = String
type XhtmlString = String

-- | Overall data model for the runtime.  
data Model = Model { -- | 
                     by_permatitle :: M.Map String Item,
                     by_int_id :: M.Map Int Item,
                     child_map :: M.Map Int [Int],
                     all_items :: [Item],
                     next_id :: Int }

empty :: Model
empty = Model M.empty M.empty M.empty [] 0

data Kind = Post | Comment | Trackback
            deriving (Show, Read, Eq)

build_model :: [Item] -> Model
build_model [] = empty
build_model items = Model (map_by permatitle sorted_items)
                    bid                    
                    (build_child_map sorted_items)
                    (sorted_items)
                    (n+1)
    where
      sorted_items = sort_by_created_reverse items
      bid = (map_by internal_id sorted_items)
      n = fst . M.findMax $ bid

build_child_map :: [Item] -> M.Map Int [Int]
build_child_map i = build_child_map_ (M.fromList $ (map (\x -> (internal_id x,[])) i)) i

-- Constructed to take advantage of the input being in sorted order.
build_child_map_ :: M.Map Int [Int] -> [Item] -> M.Map Int [Int]
build_child_map_ m [] = m
build_child_map_ m (i:is) = if (parent i == Nothing) then
                                build_child_map_ m is
                            else
                                build_child_map_ (M.insertWith (++) (unwrap $ parent i) [internal_id i] m) is

-- | Insert an item, presuming that all of its data other than
-- internal identifier have been correctly set.
insert :: Model -> Item -> (Item,Model)
insert m i = (i', m { by_permatitle = M.insert (permatitle i') i' $ by_permatitle m
                    , by_int_id = M.insert n i' $ by_int_id m
                    , child_map = M.insert (internal_id i') [] $
                                  case parent i of
                                    Nothing ->
                                        child_map m
                                    (Just p_id) ->
                                        M.insert p_id (insert_comment_ m (item_by_id m p_id) i') $ child_map m
                    , all_items = insert_ after (all_items m) i'
                    , next_id = n + 1 } )
    where
      n = next_id m
      i' = i { internal_id = n }

insert_comment_ :: Model -> Item -> Item -> [Int]
insert_comment_ m p c = map internal_id (insert_ before (children m p) c)

insert_ :: (Item -> Item -> Bool) -> [Item] -> Item -> [Item]
insert_ _ [] y = [y]
insert_ o s@(x:xs) y = if (x `o` y) then
                           (x:(insert_ o xs y))
                       else
                           (y:s)

after :: Item -> Item -> Bool
after a b = (created a) > (created b)

before :: Item -> Item -> Bool
before a b = (created a) < (created b)

-- | Apply a structure-preserving function, i.e., one that does not
-- change parent/child relationships or ids, to a specific item.
alter :: (Item -> Item) -> Model -> Item -> IO Model
alter f m i = do { ts <- now
                 ; let i' = (f i) { updated = ts }
                 ; return $ m { by_permatitle = M.insert (permatitle i') i' $ by_permatitle m
                              , by_int_id = M.insert (internal_id i') i' $ by_int_id m
                              , child_map = if (parent i == Nothing) then
                                                child_map m
                                            else
                                                M.insert p_id resort_siblings $ child_map m
                              , all_items = insert_ after all_but i' } }
    where
      not_i = \item -> (internal_id item) /= (internal_id i)
      all_but = filter not_i $ all_items m
      p_id = unwrap $ parent i
      p = item_by_id m p_id
      resort_siblings = map internal_id (insert_ before (filter not_i $ children m p) i)

cloak :: Model -> Item -> IO Model
cloak = alter (\i -> i { visible = False })

uncloak :: Model -> Item -> IO Model
uncloak = alter (\i -> i { visible = True })

permatitle_exists :: Model -> String -> Bool
permatitle_exists = (flip M.member) . by_permatitle

max_id :: Model -> Int
max_id = fst . M.findMax . by_int_id

post_by_permatitle :: Model -> String -> Item
post_by_permatitle = (!) . by_permatitle

maybe_post_by_permatitle :: Model -> String -> Maybe Item
maybe_post_by_permatitle = (flip M.lookup) . by_permatitle

item_by_id :: Model -> Int -> Item
item_by_id = (!) . by_int_id

children :: Model -> Item -> [Item]
children m i = map (item_by_id m) ((child_map m) ! (internal_id i))

unwrap :: Maybe a -> a
unwrap (Just x) = x
unwrap Nothing = error "Can't unwrap nothing!"

data Author = Author { name :: String,
                       uri :: Maybe String,
                       email :: Maybe String,
                       show_email :: Bool
                     }
              deriving ( Show,Read,Eq )

-- | General purpose runtime data structure for holding a post or
-- comment.  For a comment, a number of the fields will be ignored
-- (e.g., comments and tags) until/if the presentation and syndication
-- system gets fancier.
data Item = Item { -- | an internal unique number for this post
                   internal_id :: Int,
                   -- | the kind of item that this represents
                   kind :: Kind,
                   -- | the title of the post, as it should be rendered on
                   -- the web or inserted in an Atom feed; this should be a
                   -- valid XHTML fragment.
                   title :: XhtmlString,
                   -- | the summary of the post, as it should be rendered on
                   -- the web or intersted into an Atom feed; this should be
                   -- a valid XHTML fragment.
                   summary :: Maybe XhtmlString,
                   -- | the body of the post as an XHTML fragment.  This
                   -- will be wrapped in an XHTML @<div>@ when rendered on
                   -- the web or in a feed.
                   body :: XhtmlString,
                   -- | tags for the post, if any, expected to be in
                   -- alphabetical order and consisting of letters, digits,
                   -- dashes, and/or underscores.
                   tags :: [String],
                   -- | a generated UID for the post; this is expected to be
                   -- suitable for use as an Atom GUID.  The expectation is
                   -- that it will be supplied by the implementation when
                   -- the post is ingested.
                   uid :: String,
                   -- | a permanent title for the item, consisting of only
                   -- lowercase letters, digits, and dashes.
                   permatitle :: String,
                   -- | the timestamp, as an ISO8601 datetime, when the post
                   -- came into being.  This is never blank and would be
                   -- supplied by the implementation when the post is
                   -- ingested.
                   created :: ISO8601DatetimeString,
                   -- | the timestamp, as an ISO8601 datetime, when the post
                   -- was updated.  Initially, this is equal to the value of
                   -- the 'created' field.
                   updated :: ISO8601DatetimeString,
                   -- | the author of the post, expected to be hardwired to
                   -- the author of the blog
                   author :: Author,
                   -- | whether or not the item is to be displayed.
                   visible :: Bool,
                   -- | this item's parent, if any.
                   parent :: Maybe Int
                 }
            deriving ( Show, Read, Eq )

-- | Compute a permalink for the item relative to the supplied base URL.
permalink :: Model
          -> Item -- ^ the item
          -> String
permalink m i = U.post (relative_url m i)

relative_url :: Model -> Item -> String
relative_url m = _form_permalink . (ancestors m)

_form_permalink :: [Item] -> String
_form_permalink [] = ""
_form_permalink [i] =  let s = permatitle i in 
                       if (kind i == Post) then
                           "/" ++ s
                       else
                           "#" ++ s
_form_permalink (i:is) = if (kind i == Post) then
                            ("/" ++ permatitle i) ++ (_form_permalink is)
                        else 
                            (_form_permalink is)


ancestor_path :: Model -> Item -> String
ancestor_path m i = concat . (intersperse "/") . (map permatitle) $ ancestors m i

ancestors :: Model -> Item -> [Item]
ancestors m i = ancestors_ m [] (Just $ internal_id i)

ancestors_ :: Model -> [Item] -> Maybe Int -> [Item]
ancestors_ _ is Nothing = is
ancestors_ m is (Just i) = ancestors_ m (i':is) (parent i')
    where
      i' = item_by_id m i

lastUpdated :: [Item] -> ISO8601DatetimeString
lastUpdated ps = maximum (map updated ps)

drop_invisible :: [Item] -> [Item]
drop_invisible = filter visible

sort_by_created :: [Item] -> [Item]
sort_by_created = sortBy created_sort

created_sort :: Item -> Item -> Ordering
created_sort a b = compare (created a) (created b)

sort_by_created_reverse :: [Item] -> [Item]
sort_by_created_reverse = sortBy created_sort_reverse

created_sort_reverse :: Item -> Item -> Ordering
created_sort_reverse a b = compare (created b) (created a)

-- | Filter a list of items according to a date fragment
date_fragment_filter_ :: ISO8601DatetimeString -> [Item] -> [Item]
date_fragment_filter_ s = filter ((s `isPrefixOf`) . created)

-- | Filter a list of posts for those made in a specific year.
year_filter :: Int -- ^ year
            -> [Item] -> [Item]
year_filter y = date_fragment_filter_ $ show y

-- | Filter a list of posts for those made in a specific month.              
month_filter :: Int -- ^ year
             -> Int -- ^ month
             -> [Item] -> [Item]
month_filter y m | (0 < m) && (m < 13) = date_fragment_filter_ ((show y) ++ (pad_ m))
                 | otherwise = const []

-- | Filter a list of posts for those made on a specific day
day_filter :: Int -- ^ year
           -> Int -- ^ month
           -> Int -- ^ day
           -> [Item] -> [Item]
day_filter y m d = date_fragment_filter_ ((show y) ++ (pad_ m) ++ (pad_ d))

-- | Utility function to zero pad months and days in date expressions.
pad_ :: Int -> String
pad_ i | i < 10 = "-0" ++ (show i)
       | otherwise = ('-':(show i))
              
-- to do: make this faster using the sortedness.
tags_filter :: [String] -> [Item] -> [Item]
tags_filter t p = foldl (flip ($)) p (map tag_filter t)

tag_filter :: String -> [Item] -> [Item]
tag_filter t = filter ((t `elem`) . tags) 

plink_filterf :: String -> Item -> Bool
plink_filterf = flip $ (==) . permatitle

plink_filter :: String -> [Item] -> [Item]
plink_filter = filter . plink_filterf

ymd_plink_finder :: Int -> Int -> Int -> String -> [Item] -> [Item]
ymd_plink_finder y m d t = (plink_filter t) . (day_filter y m d)

all_posts :: Model -> [Item]
all_posts = (filter (\x -> Post == kind x)) . all_items

all_comments :: Model -> [Item]
all_comments = (filter (\x -> Comment == kind x)) . all_items

flatten :: Model -> [Item] -> [Item]
flatten m = flatten_ (children m)

flatten_ :: (a -> [a]) -> [a] -> [a]
flatten_ _ [] = []
flatten_ f (i:is) = (i:(flatten_ f (f i))) ++ (flatten_ f is)

concat_comments :: Model -> [Item] -> [Item]
concat_comments m = (foldr (++) []) . (map $ children m)

(</>) :: String -> String -> String
s </> t = s ++ ('/':t)

to_string :: Item -> String
to_string i = concat [metadata i, "\n", body_block i, "\n", summary_block i]

metadata :: Item -> String
metadata i = unlines $ apply i [ ("internal_id",show . internal_id),
                                 ("parent", show . parent),
                                 ("title",title),
                                 ("tags",show_no_quotes . tags),
                                 ("permatitle",permatitle),
                                 ("kind",show . kind),
                                 ("uid",uid),
                                 ("created",created),
                                 ("updated",updated),
                                 ("author",show . author),
                                 ("visible",show . visible) ]

show_no_quotes :: [String] -> String
show_no_quotes = concat . (intersperse ", ")

apply :: Item -> [(String,(Item -> String))] -> [String]
apply _ [] = []
apply i (x:xs) = ((concat [fst x, ": ", (snd x) i]) : (apply i xs))

body_block :: Item -> String
body_block i = concat ["--- START BODY ---\n",
                       (body i),
                       "\n--- END BODY ---\n"]

summary_block :: Item -> String
summary_block i | summary i == Nothing = ""
                | otherwise = concat ["--- START SUMMARY ---\n",
                                      (unwrap $ summary i),
                                      "\n--- END SUMMARY ---\n"]

default_author :: Author
default_author = Author C.author_name C.author_uri C.author_email True

In Scala:

• Formatted and highlighted by NetBeans Scala Plugin, exported via [File] -> [Print to HTML ...]

package org.aiotrade.blog.model

import java.util.Calendar
import org.aiotrade.blog.{Constants => C}

object Entry {
  type XhtmlString = String

  class Model (
    var by_permatitle: Map[String, Item],
    var by_int_id: Map[Int, Item],
    var child_map: Map[Int, List[Int]],
    var all_items: List[Item],
    var next_id: Int
  ) {
    // call by name
    def apply(block: => Unit) = {block; this}
  }

  abstract class Kind //deriving (Show, Read, Eq)
  case object Post extends Kind
  case object Comment extends Kind
  case object Trackback extends Kind

  case class Author (
    var name: String,
    var uri : Option[String] = None,
    var email: Option[String] = None,
    var show_email: Boolean = false
  )

  /** General purpose runtime data structure for holding a post or
   * comment.  For a comment, a number of the fields will be ignored
   * (e.g., comments and tags) until/if the presentation and syndication
   * system gets fancier.
   */
  case class Item (
    // an internal unique number for this post
    var internalId: Int,
    // the kind of item that this represents
    var kind: Kind,
    // the title of the post, as it should be rendered on
    // the web or inserted in an Atom feed; this should be a
    // valid XHTML fragment.
    var title: XhtmlString,
    // the summary of the post, as it should be rendered on
    // the web or intersted into an Atom feed; this should be
    // a valid XHTML fragment.
    var summary: Option[XhtmlString],
    // the body of the post as an XHTML fragment.  This
    // will be wrapped in an XHTML @<div>@ when rendered on
    // the web or in a feed.
    var body: XhtmlString,
    // tags for the post, if any, expected to be in
    // alphabetical order and consisting of letters, digits,
    // dashes, and/or underscores.
    var tags: List[String],
    // a generated UID for the post; this is expected to be
    // suitable for use as an Atom GUID.  The expectation is
    // that it will be supplied by the implementation when
    // the post is ingested.
    var uid: String,
    // a permanent title for the item, consisting of only
    // lowercase letters, digits, and dashes.
    var permatitle: String,
    // the timestamp, as an ISO8601 datetime, when the post
    // came into being.  This is never blank and would be
    // supplied by the implementation when the post is
    // ingested.
    var created: Long,
    // the timestamp, as an ISO8601 datetime, when the post
    // was updated.  Initially, this is equal to the value of
    // the 'created' field.
    var updated: Long,
    //the author of the post, expected to be hardwired to
    // the author of the blog
    var author: Author,
    //whether or not the item is to be displayed.
    var visible: Boolean,
    //this item's parent, if any.
    var parent: Option[Int]
  ) {
    def apply(block: Item => Unit) = {block(this); this}
  }

  def empty = new Model(Map(), Map(), Map(), Nil, 0)

  def build_model(is: List[Item]) =
    is match {
      case Nil => empty
      case _ =>
        val sortedIs = sort_by_created_reverse(is)
        val bid = Map() ++ sortedIs.map{x => (x.internalId -> x)}
        val n = bid.keySet.max

        new Model(Map() ++ sortedIs.map{x => (x.permatitle -> x)},
                  bid,
                  buildChildMap(sortedIs),
                  sortedIs,
                  n + 1)
    }

  def buildChildMap(is: List[Item]) =
    buildChildMap_(Map() ++ is.map(_.internalId -> Nil), is)

  def buildChildMap_(map: Map[Int, List[Int]], is: List[Item]) =
    map ++ {
      for (i <- is if i.parent.isDefined) yield {
        // pid, cids definitions go into body // it's more efficient.
        val pid = i.parent.get
        val cids = map.getOrElse(pid, Nil)
        pid -> (i.internalId :: cids)
      }
    }

  /** Insert an item, presuming that all of its data other than
   internal identifier have been correctly set.
   */
  def insert(m: Model, i: Item): (Item, Model) = {
    val n = m.next_id
    i.internalId = n

    (i, m {
        m.by_permatitle += (i.permatitle -> i)
        m.by_int_id += (n -> i)
        m.child_map = (i.parent match {
            case None => m.child_map
            case Some(p_id) => m.child_map + (p_id -> (insert_comment_(m, item_by_id(m)(p_id), i)))
          })
        m.all_items = insert_(after, m.all_items, i)
        m.next_id = n + 1 }
    )
  }

  def insert_comment_(m: Model, p: Item, c: Item): List[Int] = 
    insert_(before, children(m)(p), c) map (_.internalId)

  def insert_(o: (Item, Item) => Boolean, is: List[Item], y: Item): List[Item] =
    is match {
      case Nil => List(y)
      case x :: xs => if (o(x, y)) x :: insert_(o, xs, y) else (y :: is)
    }
  
  def after (a: Item, b: Item): Boolean =
    a.created > b.created

  def before(a: Item, b: Item): Boolean =
    a.created < b.created

  /**
   * Apply a structure-preserving function, i.e., one that does not
   * change parent/child relationships or ids, to a specific item.
   */
  def alter(f: (Item => Item), m: Model, i: Item): Model = {// -> IO Model
    val not_i = (item: Item) => item.internalId != i.internalId
    val all_but = m.all_items filter not_i
    val p_id = unwrap (i.parent)
    val p = item_by_id(m)(p_id)
    val resort_siblings = insert_(before, children(m)(p) filter not_i, i) map (_.internalId)

    val ts = System.currentTimeMillis
    val i1 = f(i) {_.updated = ts}
    m {
      m.by_permatitle += (i1.permatitle -> i1)
      m.by_int_id += (i1.internalId -> i1)
      m.child_map = i.parent match {
        case None => m.child_map
        case _ => m.child_map + (p_id -> resort_siblings)
      }
      m.all_items = insert_(after, all_but, i1)
    }
  }

  def cloak(m: Model, i: Item): Model = // -> IO Model
    alter (i => i {_.visible = false}, m, i)

  def uncloak(m: Model, i: Item): Model = // -> IO Model
    alter (i => i {_.visible = true}, m, i)

  def permatitle_exists(m: Model, p: String): Boolean =
    m.by_permatitle.contains(p)

  def max_id(m: Model): Int =
    m.by_int_id.keySet.max

  def post_by_permatitle(m: Model, p: String): Item =
    m.by_permatitle(p)

  def maybe_post_by_permatitle(m: Model, p: String): Option[Item] =
    m.by_permatitle.get(p)

  def item_by_id(m: Model)(id: Int): Item = 
    m.by_int_id(id)

  def children(m: Model)(i: Item): List[Item] =
    m.child_map(i.internalId) map (item_by_id(m))

  def unwrap[T](a: Option[T]): T =
    a match {
      case Some(x) => x
      case None => error("Can't unwrap none!")
    }

  def relative_url(m: Model, i: Item): String =
    _form_permalink(ancestors(m, i))

  def _form_permalink(is: List[Item]): String =
    is match {
      case Nil => ""
      case i :: Nil =>
        val s = i.permatitle
        if (i.kind == Post)
          "/" + s
        else
          "#" + s
      case i :: is =>
        if (i.kind == Post)
          ("/" + i.permatitle) + _form_permalink(is)
        else
          _form_permalink(is)
    }

  def ancestors(m: Model, i: Item): List[Item] =
    ancestors_(m, Nil, Some(i.internalId))

  def ancestors_(m: Model, is: List[Item], i_? : Option[Int]): List[Item] =
    i_? match {
      case None => is
      case Some(i) =>
        val i1 = item_by_id(m)(i)
        ancestors_(m, i1 :: is, i1.parent)
    }

  def lastUpdated(ps: List[Item]): Long =
    ps map (_.updated) max

  def drop_invisible(is: List[Item]): List[Item] =
    is filter (_.visible)

  def sort_by_created(is: List[Item]): List[Item] =
    is sortWith created_sort _

  def created_sort(a: Item, b: Item) =
    a.created < b.created

  def sort_by_created_reverse(is: List[Item]): List[Item] =
    is sortWith created_sort_reverse _

  def created_sort_reverse(a: Item, b: Item) =
    b.created < a.created

  def date_fragment_filter_(is: List[Item], ts: Int*) = {
    val cal = Calendar.getInstance
    ts match {
      case Seq(y, m, d) => is filter {i => pad_(cal, i.created) match {
            case (`y`, `m`, `d`) => true
            case _ => false
          }
        }
      case Seq(y, m) => is filter {i => pad_(cal, i.created) match {
            case (`y`, `m`, _) => true
            case _ => false
          }
        }
      case Seq(y) => is filter {i => pad_(cal, i.created) match {
            case (`y`, _, _) => true
            case _ => false
          }
        }
    }
  }
  
  def year_filter(y: Int)(is: List[Item]): List[Item] =
    date_fragment_filter_(is, y)

  def month_filter(y: Int, m: Int)(is: List[Item]): List[Item] =
    date_fragment_filter_(is, y, m)

  def day_filter(y: Int, m: Int, d: Int)(is: List[Item]): List[Item] =
    date_fragment_filter_(is, y, m, d)

  def pad_(cal: Calendar, t: Long): (Int, Int, Int) = {
    cal.setTimeInMillis(t)
    import Calendar._
    (cal.get(YEAR), cal.get(MONTH) + 1, cal.get(DAY_OF_MONTH))
  }

  def tags_filter(ts: List[String])(is: List[Item]): List[Item] =
    is filter (i => (false /: ts) {_ || i.tags.contains(_)})

  def tag_filter(t: String)(is: List[Item]): List[Item] =
    is filter (_.tags.contains(t))

  def plink_filterf(p: String)(i: Item): Boolean =
    i.permatitle == p

  def plink_filter(p: String)(is: List[Item]): List[Item] =
    is filter plink_filterf(p)

  def ymd_plink_finder(y: Int, m: Int, d: Int, p: String)(is: List[Item]): List[Item] =
    plink_filter(p) (day_filter(y, m, d) (is))

  def </> (s: String, t: String): String =
    s + '/' + t

  def all_posts(m: Model): List[Item] =
    m.all_items filter (_.kind == Post)

  def all_comments(m: Model): List[Item] =
    m.all_items filter (_.kind == Comment)

  def flatten(m: Model, is: List[Item]): List[Item] =
    flatten_(children(m), is)

  def flatten_[T](f: T => List[T], is: List[T]): List[T] =
    is match {
      case Nil => Nil
      case i :: is => i :: flatten_(f, f(i)) ::: flatten_(f, is)
    }

  def concat_comments(m: Model, is: List[Item]): List[Item] =
    ((is map children(m)) :\ List[Item]())(_ ::: _)

  def metadata(i: Item): String =
    apply(i, List(("internal_id", show(_.internalId)),
                  ("parent",      show(_.parent)),
                  ("title",       _.title),
                  ("tags",        show_no_quotes(_.tags)),
                  ("permatitle",  _.permatitle),
                  ("kind",        show(_.kind)),
                  ("uid",         _.uid),
                  ("created",     show(_.created)),
                  ("updated",     show(_.updated)),
                  ("author",      show(_.author)),
                  ("visible",     show(_.visible)))
    ) mkString "\n"

  // curring
  def show(f: Item => Any)(i: Item): String =
    f(i) toString

  def show_no_quotes(f: Item => List[String])(i: Item): String =
    f(i) mkString ", "

  def apply(i: Item, xs: List[(String, Item => String)]): List[String] =
    xs match {
      case Nil => Nil
      case x :: xs => (x._1 + ": " + x._2(i)) :: apply(i, xs)
    }

  def body_block(i: Item): String =
    "--- START BODY ---\n" + i.body + "\n--- END BODY ---\n"

  def summary_block(i: Item): String =
    i.summary match {
      case None => ""
      case Some(x) => "--- START SUMMARY ---\n" + x + "\n--- END SUMMARY ---\n"
    }

  val default_author = Author(C.author_name, C.author_uri, C.author_email, true)
}

My next step is to push AIOTrade to another level of UE, with hotkey for symbols, indicators etc, with better UI.

So far, my Scala adventure went almost smoothly, I'm thinking about the data exchange between client and data source, maybe the actors from  Scala,  LiftWeb or  AKKA will bring some fresh atmosphere.

Well, I've done most parts of migrating AIOTrade to Scala, not all features return yet. I gain lots of experiences of inter-op between Scala and Java, since AIOTrade has to be integrated into an existed Java framework  NetBeans Platform. And also, whole project is now managed by Maven instead of Ant, which reduces lots of pain of dependencies upon crossing sub-projects.

This project is now hosted on  kenai.com  http://sf.net/projects/humaitrader, you can check out the code to get an overview of how to integrated Maven + Scala + NetBeans Modules. Of course, all were done with NetBeans Scala plugin.

LOC of this project so far:

$ ./cloc.pl --read-lang-def=lang_defs.txt ~/myprjs/aiotrade.kn/opensource/
     677 text files.
     617 unique files.                                          
     154 files ignored.

http://cloc.sourceforge.net v 1.08  T=3.0 s (167.7 files/s, 21373.7 lines/s)
-------------------------------------------------------------------------------
Language          files     blank   comment      code    scale   3rd gen. equiv
-------------------------------------------------------------------------------
Scala               353      7981     16301     27180 x   1.36 =       36964.80
Java                 43      1148       833      6946 x   1.36 =        9446.56
XML                 104       231       389      2414 x   1.90 =        4586.60
Bourne Shell          2        81        81       488 x   3.81 =        1859.28
HTML                  1         7        15        26 x   1.90 =          49.40
-------------------------------------------------------------------------------
SUM:                503      9448     17619     37054 x   1.43 =       52906.64
-------------------------------------------------------------------------------

A screen snapshot:

I ported  jfastcgi to Scala and fixed some bugs. Now my web site is configured running under Jetty with the modified fastcgi servlet as gateway to a Trac fastcgi daemon.

I encountered an issue (fixed later) with Trac fcgi daemon though, which may cause the daemon died silently, but the fastcgi servlet will restart it automatically when this happens.

All source code can be found via Browse Source

I've almost done importing old blogs from Roller db to this Trac based one, except image/resource links.

The pretty url of this web site is also working now.

I just re-launched blogtrader.net, which is now based on trac, but on glassfish V3.0 (moved to Jetty 7.0 later) via a custom CGIServlet. It's a headache to get a pretty url for trac, I may re-explore it later.

All old blogs should be imported back here, but not now.

Caoyuan's blogs about future.

For more information about this site see Wiki

I have to say, maintain a couple of dependent projects via Ant is a headache, just like me and others may have encountered when use NetBeans' Scala plugin to create and maintain plain Scala projects, these projects, are all Ant based. It's difficult to write a generic Ant template to compute the dependencies graph and then choose the best building path. The building process of cross dependent projects becomes very slow because the redundant enter/exit dependent projects building task.

NetBeans has also the best Maven plugin integrated, I decided to give Maven a trying. This was actually my first attempt to create Maven based project(s). I explored the mini way toward a Scala Maven project, and patched Scalac a bit again to get the dependencies aware compiling working.

Now here's an example I'd like to share with you.

Assume we have n sub-projects, with cross dependencies, for instance, 'lib.util' and 'lib.math', 'lib.indicators' etc. The simplest Maven way is to keep a parent project (I call it 'modules' here) which holds all common settings and module(sub-project) names(paths). The directory structure could be like:

modules
|-- pom.xml
|-- lib.util
|   |-- pom.xml
|   `-- src
|       |-- main
|       |   `-- scala
|       |       `-- lib
|       |           `-- util
|       |               `-- App.scala
|       `-- test
|           `-- scala
|               `-- lib
|                   `-- util
|                       `-- AppTest.scala
|-- lib.math
    |-- pom.xml
    `-- src
        |-- main
        |   `-- scala
        |       `-- lib
        |           `-- math
        |               `-- App.scala
        `-- test
            `-- scala
                `-- lib
                    `-- math
                        `-- AppTest.scala

What I like Maven here is that the parent project 'modules' only maintains the common settings and module paths, all dependencies between sub-projects are then set in sub-project's self pom.xml file. This is a good decoupled strategy in my eyes, you can freely combine these sub-projects at any time without change too much, and each sub-project doesn't need to care about the dependencies of other projects. The parent project is only a centre place for sharing common setting and a centre place to list all available projects and their directories, it's just like a Directory Service.

Now, here is the pom.xml of parent project looks like (all common settings include compiler, repository etc are kept in it):

<project xmlns="http://maven.apache.org/POM/4.0.0"
         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" 
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">

    <modelVersion>4.0.0</modelVersion>

    <groupId>org.aiotrade</groupId>
    <artifactId>modules</artifactId>
    <version>1.0-SNAPSHOT</version>
    <packaging>pom</packaging>

    <name>AIOTrade Modules</name>
    <description>Basic Modules for AIOTrade.</description>

    <properties>
        <scala.version>2.8.0-SNAPSHOT</scala.version>
    </properties>

    <repositories>
        <repository>
            <id>scala-tools.release</id>
            <name>Scala-Tools Release Repository</name>
            <url>http://scala-tools.org/repo-releases</url>
        </repository>
        <repository>
            <id>scala-tools.snapshot</id>
            <name>Scala-Tools Snapshot Repository</name>
            <url>http://scala-tools.org/repo-snapshots</url>
        </repository>
    </repositories>

    <pluginRepositories>
        <pluginRepository>
            <id>scala-tools.org</id>
            <name>Scala-Tools Maven2 Repository</name>
            <url>http://scala-tools.org/repo-releases</url>
        </pluginRepository>
    </pluginRepositories>

    <dependencies>
        <dependency>
            <groupId>org.scala-lang</groupId>
            <artifactId>scala-library</artifactId>
            <version>${scala.version}</version>
        </dependency>
        <dependency>
            <groupId>junit</groupId>
            <artifactId>junit</artifactId>
            <version>4.4</version>
            <scope>test</scope>
        </dependency>
        <dependency>
            <groupId>org.specs</groupId>
            <artifactId>specs</artifactId>
            <version>1.2.5</version>
            <scope>test</scope>
        </dependency>
    </dependencies>

    <build>
        <plugins>
            <plugin>
                <groupId>org.scala-tools</groupId>
                <artifactId>maven-scala-plugin</artifactId>
                <executions>
                    <execution>
                        <phase>process-resources</phase> <!-- to support mix java/scala only -->
                        <goals>
                            <goal>add-source</goal>      <!-- to support mix java/scala only -->
                            <goal>compile</goal>
                            <goal>testCompile</goal>
                        </goals>
                    </execution>
                </executions>
                <configuration>
                    <jvmArgs>
                        <jvmArg>-Xms64m</jvmArg>
                        <jvmArg>-Xmx1024m</jvmArg>
                    </jvmArgs>
                    <scalaVersion>${scala.version}</scalaVersion>
                    <args>
                        <arg>-target:jvm-1.5</arg>
                        <arg>-make:transitivenocp</arg>
                        <arg>-dependencyfile</arg>
                        <arg>${project.build.directory}/.scala_dependencies</arg>
                    </args>
                </configuration>
            </plugin>
            <plugin>
                <groupId>org.apache.maven.plugins</groupId>
                <artifactId>maven-compiler-plugin</artifactId>
                <configuration>
                    <source>1.5</source>
                    <target>1.5</target>
                </configuration>
            </plugin>
        </plugins>
    </build>

    <reporting>
        <plugins>
            <plugin>
                <groupId>org.scala-tools</groupId>
                <artifactId>maven-scala-plugin</artifactId>
                <configuration>
                    <scalaVersion>${scala.version}</scalaVersion>
                </configuration>
            </plugin>
        </plugins>
    </reporting>
    
    <modules>
        <module>lib.math</module>
        <module>lib.util</module>
        <module>lib.indicator</module>
    </modules>
</project>

And the pom.xml of sub-project 'lib.util' looks like:

<project xmlns="http://maven.apache.org/POM/4.0.0"
         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" 
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">

    <modelVersion>4.0.0</modelVersion>

    <parent>
        <groupId>org.aiotrade</groupId>
        <artifactId>modules</artifactId>
        <version>1.0-SNAPSHOT</version>
    </parent>

    <artifactId>lib-util</artifactId>
    <version>1.0-SNAPSHOT</version>
    <name>lib-util</name>

</project>

The 'lib.math' one:

<project xmlns="http://maven.apache.org/POM/4.0.0"
         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" 
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">

    <modelVersion>4.0.0</modelVersion>

    <parent>
        <groupId>org.aiotrade</groupId>
        <artifactId>modules</artifactId>
        <version>1.0-SNAPSHOT</version>
    </parent>

    <artifactId>lib-math</artifactId>
    <version>1.0-SNAPSHOT</version>
    <name>lib-math</name>
    
    <dependencies>
        <dependency>
            <groupId>${project.groupId}</groupId>
            <artifactId>lib.util</artifactId>
            <version>${project.version}</version>
            <type>jar</type>
        </dependency>
    </dependencies>

</project>

That's all the settings, clear and clean.

Now, let's back to another level of dependencies: dependencies between Scala source files. How does this setting get scalac aware dependencies of changed sources?

Again, just like the scalac setting in an Ant based project, where at 'configuration' part of build/plugins/plugin/@maven-scala-plugin, I add "-make:transitivenocp -dependencyfile ${project.build.directory}/.scala_dependencies" to 'args' (should be put in separate 'arg' tag as following)

                <configuration>
                    <jvmArgs>
                        <jvmArg>-Xms64m</jvmArg>
                        <jvmArg>-Xmx1024m</jvmArg>
                    </jvmArgs>
                    <scalaVersion>${scala.version}</scalaVersion>
                    <args>
                        <arg>-target:jvm-1.5</arg>
                        <arg>-make:transitivenocp</arg>
                        <arg>-dependencyfile</arg>
                        <arg>${project.build.directory}/.scala_dependencies</arg>
                    </args>
                </configuration>

Now the last question, how to use NetBeans to create above setting?

Open "File" | "New Project..." | "Maven" | "Maven Project", click "Next", choose "Maven Quickstart Archetype" to create each empty project, then copy/paste above content to corresponding pom.xml, change project groupId, artifactId and version properties. For existed project, copy your source code to "main/scala". You can also mixed Java code, put then at "main/java", NetBeans Scala plugin supports mixed Scala/Java project well.

I'll put the pre-defined archetype on net later, so no need to copy/paste, or, you can create an empty parent/sub-project, commit to your version control system. BTW, I find manually modify pom.xml is a pleasure work.

• Note No. 1: You have to use/update-to Scala-2.8.0-SNAPHOST with at least scala-compiler-2.8.0-20091128-xxx.jar

• Note No. 2: If your project is simple, just use NetBeans created Ant based project. If you separate it to a couple of dependent projects, I strongly suggest you to Maven.

• Note No. 3: A new Scala plugin for NetBeans 6.8RC1 will be available before or at Monday, it's also a must upgrade especially for Windows users.

During the past days, I was patching scalac ant task and some relative issues, and now, the dependencies aware ant scalac works (post Scala 2.8.0.r19724).

Below is an example build.xml with dependencies aware setting:

 1 <?xml version="1.0" encoding="UTF-8"?>
 2 <project name="ScalaAntTest" default="build" basedir=".">
 3     <description>Builds, tests, and runs the project ScalaAntTest.</description>
 4 
 5     <property name="src.dir" value="${basedir}/src"/>
 6     <property name="build.dir" value="${basedir}/build"/>
 7     <property name="build.classes.dir" value="${build.dir}/classes"/>
 8 
 9     <target name="init">
10         <property environment="env"/>
11         <condition property="scala.home" value="${env.SCALA_HOME}">
12             <isset property="env.SCALA_HOME"/>
13         </condition>
14         <fail unless="scala.home">set SCALA_HOME first</fail>
15 
16         <property name="scala-library.jar" value="${scala.home}/lib/scala-library.jar"/>
17         <property name="scala-compiler.jar" value="${scala.home}/lib/scala-compiler.jar"/>
18 
19         <path id="build.classpath">
20             <pathelement location="${scala-library.jar}"/>
21             <pathelement location="${scala-compiler.jar}"/>
22             <pathelement location="${build.classes.dir}"/>
23         </path>
24         <taskdef resource="scala/tools/ant/antlib.xml">
25             <classpath>
26                 <pathelement location="${scala-compiler.jar}"/>
27                 <pathelement location="${scala-library.jar}"/>
28             </classpath>
29         </taskdef>
30     </target>
31 
32     <target name="build" depends="init">
33         <mkdir dir="${build.dir}"/>
34         <mkdir dir="${build.classes.dir}"/>
35         <scalac srcdir="${src.dir}"
36                 destdir="${build.classes.dir}"
37                 classpathref="build.classpath"
38                 force="yes"
39                 addparams="-make:transitive -dependencyfile ${build.dir}/.scala_dependencies"
40                 >
41             <src path="${basedir}/src1"/> 
42             <!--include name="compile/**/*.scala"/-->
43             <!--exclude name="forget/**/*.scala"/-->
44         </scalac>
45     </target>
46 
47     <target name="clean" depends="init">
48         <delete dir="${build.dir}"/>
49     </target>
50 
51 </project>

There are some tips here, I'll give a concise explanation:

First, there will be a file call ".scala_dependencies" which is put under "build/" directory after you first clean-build, it will record all dependencies information. Since it's put under "build/", it will be removed automatically after an "ant clean". The "-dependencyfile ${build.dir}/.scala_dependencies" parameter of scalac at line 39 enables this.

Second, you should add "-make:transitive" as scalac's parameter (line 39), which will enable scalac to evaluate the dependencies transitive.

Third, add attribute "force='yes'" (line 38), which tells scalac to check all source files for dependencies and re-compile them if files that dependents on changed.

Forth, you should include "<pathelement location='${build.dir.classes}'>" as part of "build.classpath" (line 22), so scalac won't complain lack of already generated classes when rebuild upon parts of source files.

I've also re-write the project scheme that created by NetBeans plugin, that is, the next released NetBeans Scala plugin will automatically generate dependencies aware build.xml file for new created projects. Unfortunately, you'll have to copy/move your old project src to new created project directory if you want to benefit from it.

For some reasons, "fsc" is no longer supported as the default compiler task in NetBeans created project, which actually brought some annoyances for plugin users. Even without "fsc", the dependencies aware "scalac" should also work satisfiable in most cases.

I packed another release of Scala plugin for NetBeans 6.8 beta, for more information, please see:

 http://wiki.netbeans.org/Scala68v1

This version allows adding "deprecation", "unchecked" parameters for scalac, and fixed unicode id issue. It works on Scala-2.8.0 snapshot only.

Note:

All projects created via "File" -> "New Project" before this version should add or change following lines in "nbproject/project.properties":

scalac.compilerargs=
scalac.deprecation=no
scalac.unchecked=no

As NetBeans 6.8 m2 released, I packed a downloadable binary which works with Scala 2.8.0.r18993 or above.

Please do not install Scala plugin via Update Center for NetBeans 6.8 m2, it's not compatible. Use the link below to download and install

New & Noteworthy

• Much more responsive when typing and for code-completion
• More refactoring: find usages, rename across opened projects
• Breakpoint works everywhere (almost)
• Better supporting for mixed Java/Scala project in both direction (Java is visible in Scala and vice versa)
• Better integration with NetBeans maven plugin
• Better code completion even for implicit methods (in most cases)
• Better indentation and formatter for unfinished line after 'if', 'else', '=', 'for', 'while', 'do' etc
• Better syntax highlighting for val/var, lazy val, implicit call, byname param, abstract method etc
• Mark of implemented/overridden methods, click on mark will jump to super definition
• Go to type ("Ctrl+O")
• Select parts of code, press '{', '[', '(', '"', '`' will add/replace surrounding pair, press '~' will remove them. Specially, press '/' will block-comment it
• Reset Scala interactive parser when necessary, for instance: dependent libs changed (Right click on source, choose "Reset Scala Parser" in pop-up menu)
• Output highlighted code to html ([File] -> [Print to HTML...])
• Some basic hints, for instance: fixing import, unused imports
• Code template now works, check or add your own via [Options/Preferences] -> [Editor] -> [Code Templates] -> [Scala]

Install with NetBeans 6.8 M2

1. Download and install the latest Scala-2.8.0 snapshot runtime via Scala's home
2. Set $SCALA_HOME environment variable to point to the installed Scala runtime path. Add $SCALA_HOME/bin to PATH environment variable. Note for Mac OS user, $SCALA_HOME environment variable may not be visible for Applications/NetBeans, see  http://wiki.netbeans.org/MacOSXEnvForApp
3. Get the NetBeans 6.8 M2 from:  http://bits.netbeans.org/netbeans/6.8/m2/
4. Get the Scala plugins binary from:  https://sourceforge.net/projects/erlybird/files/nb-scala/6.8v1.1.0m2/nb-scala-6.8v1.1.0m2.zip/download
5. Unzip Scala plugin binary to somewhere
6. Open NetBeans, go to "Tools" -> "Plugins", click on "Downloaded" tab title, click on "Add Plugins..." button, choose the directory where the Scala plugins are unzipped, select all listed *.nbm files, following the instructions.

Tips

• If you encounter "... Could not connect to compilation daemon.", try to run "fsc -reset" under a command/terminal window.
• Editor $NetBeansInstallationPath?/etc/netbeans.conf, remove "-ea" to avoid AssertionError? popped up

 http://wiki.netbeans.org/Scala68v1 for more information

I'm going to migrate this web site to new host during this month, both blogtrader.net and aiotrade.com may not be available at any time during this period.

>>> Updated on Sep 8:

Now in most cases, Scala is visible to Java, including generic type. Mixed Java/Scala project should be well supported.

======

I need "Go To Type" working, and Scala's symbols should be visible to Java sources. For NetBeans, that means I should implement a Scala to Java VirtualSourceProvider?, which will translate Scala source to Java stub.

I tried it before, but not so successful, so I disabled it. Finally, I got how it works today, and committed work in part. That is, now, in Scala/Java mixed project, not only Java is visible to Scala source, and also, partly, Scala is visible to Java too.

Another benefit is, when you press "Ctrl + O" or "Command + O", a navigator window will bring you to Type's source file that you are searching.

I'll go on to get Scala -> Java mapping fully works soon.

>>> Updated on Sep 8:

The following issue was fixed in trunk.

======

Warning: If you have not got a NetBeans nightly built before, do not try it until a recent (after Sep 3rd) serious fault fixed.

The new progress is a little mark at the left side bar of editor, showing if a method is overriding a method of base class/trait. Move cursor on this mark will show a tooltip, and, click on this mark will jump to source of super definition.

If a mark of (D) shown and you forget to add "override", you will wait until a whole build process to tell you lacking of "override", so pay attention to it.

Of course, if it's an (I) mark, you do not need to add "override".

I'm working on Refactoring features for NetBeans' Scala plugin, the first step is finding usages of class/method/val etc across project's source files, it works now. The next step will be renaming them.

Following snapshot shows class "ScalaDfn?" are used in 19 places, across 6 Scala sources

As I've done most code of Refactoring skeleton, it did not cost me too much time to get renaming feature working (2 hours about). Now, NetBeans' Scala plugin can rename class/method/val etc across project's source files.

Following snapshot shows class "Dog" is going to be renamed to "BigDog?". After the preview, press "Do Refactoring", all things done.

I begin to patch Scala's new designed compiler interface for IDE to get it work better (with NetBeans, and, other IDEs). I know lamp term may use Eclipse as their IDE to develop Scala itself, but I'm using new written NetBeans plugin. Here's a short tutorial of how to use NetBeans Scala plugin to work with Scala itself

Build Scala-2.8.0 snapshot

First and least, svn check out Scala's trunk source code:

cd ~/myprjs/scala/scala
svn co http://lampsvn.epfl.ch/svn-repos/scala/scala/trunk

Then, make sure your $JAVA_HOME is pointed to a JDK 1.5 instead of 1.6, and <b>clear $SCALA_HOME</b>. "ant dist" to build a fresh Scala distribution, which is located at ~myprjs/scala/scala/dist/latest

# As for Mac OS X
JAVA_HOME_BAK=$JAVA_HOME
export JAVA_HOME=/System/Library/Frameworks/JavaVM.framework/Versions/1.5.0/Home
export SCALA_HOME=
ant dist
export JAVA_HOME=$JAVA_HOME_BAK
export SCALA_HOME=~myprjs/scala/scala/dist/latest

Install latest NetBeans and Scala plugin

Download latest NetBeans nightly built from  http://bits.netbeans.org/download/trunk/nightly/latest, the minimal pack "Java SE" is enough.

Run NetBeans, get latest Scala plugin via: [Preference/Option] -> [Plugins] -> [Available Plugins], find "Scala Kit" in the list and choose it,following the instructions to get it installed, restart NetBeans.

Create NetBeans project by importing existing sources

Create project for Scala's trunk sources. <b>Note: each folder under "src" should be considered as standalone folder, for example, "compiler"</b>, it's better to create standalone project for each these folder, I created two, one for "compiler",another for "library".

[File] -> [New Project] -> [Scala] -> [Scala Project with Existing Sources]

Click [Next], input project name that you like Note: you can input an existed ant build script file name in "Build Script Name". Or, modify the auto-created one later, it's a plain ant build xml file.

Click [Next], add source folder:

Click [Next] again, you can see the sources that will be included in this project, then click [Finish]

Now, Scala's compiler has been imported as a NetBeans freeform project, it is an old plain ant project. You can create another one for Scala's library.

How to debug into Scala's compiler, library source?

If you have a project that want to debug into Scala's compiler, library sources, you could do when in debugging:

In debugging, open debugging source windows via: [Windows] -> [Debugging] -> [Sources]. Go through the listed source jars, find ".../scala-compiler.jar" and ".../scala-library.jar", check them.

I struggled with  new redesigned IDE / compiler interface (interative.Global) during this weekend, and finally, got it attached to NetBeans Scala plugin, which runs as a background daemon thread, thus the plugin is more responsive when coding (the first benefit). It was a hard work though, I had to modify some of the original code to get whole things stable and responsive (balance), it paid me 2 sleepless nights.

So here's what's new on current nightly built:

• Tested and work with Scala 2.8.0.r18542 snapshot
• Better supporting for mixed Java/Scala project
• Better indentation and formatter for unfinished line after 'if', 'else', '=', 'for', 'while', 'do' etc
• Better code completion even for implicit methods (some times)
• Implicit method call will be highlighted by underline
• Select parts of code, press '{', '[', '(', '"', '`' etc will add/replace surrounding pair, press '~' will remove them. Specially, press '/' will block comment it
• Some basic hints, for example, fix import (from Milos' work)
• Code template now works, check or add your own via [Options/Preferences] -> [Editor] -> [Code Templates] -> [Scala] (from Milos' work)

Note: wait until Monday night for NetBeans' hudson builds those whole new nbms.

I think this plugin is qualifiable for beta using/testing now. If you are interested in testing/reporting bugs, you can get the latest NetBeans nightly built, and got the new Scala plugin from Update Center, download a Scala-2.8.0 snapshot, live your $SCALA_HOME to this Scala 2.8.0 snapshot, change $PATH to including $SCALA_HOME/bin.

By testing/using Scala 2.8.0 right now, we can also push it improved more quickly.

BTW, I'm using this plugin on whole Scala' trunk source and of course, this plugin.

Java/Scala mixed project was supported partly once before, but I cannot remember when this feature got lost. Anyway, as one progressing of the rewritten NetBeans Scala plugin, I just got this feature working properly halfway: now Java source is visible to Scala one, but not vice versa. To got Scala source also visible for Java source, would pay for efficiency, since the compiler may need to be pushed to a deep phase, and mapped to Java classes/methods, I need more time to think about that.

Below is a screenshot that a JavaDog? is called from a Scala source file:

The previous Scala plugin for NetBeans was a rush work which was written in Java, toward a useful tool to support writing Scala code. I called it chicken, which was then can be used to produce new Scala plugin written in Scala, that was, the egg.

The egg has grown to a chicken now. As for last night, I switched whole scala modules to depend on the new written scala.editor module (in Scala), and use this new chicken to improve Scala plugin from now on.

The new scala plugin will be carefully rewritten, with a lot of APIs supporting for language editor in mind, and try to support better and more features toward a whole featured, stable Scala IDE.

Another good news is, Milos Kleint, the contributor to NetBeans Maven plugin, has also put hands on this project, he's working on better Scala support for maven project, and new templates/hinds features for Scala editor modules.

The new plugin is based on Scala 2.8 snapshot, if want to get it, you should use the nightly built NetBeans from netbeans.org, and get the new plugins via updated center.

Note: you have to install the latest Scala 2.8 snapshot too, and make sure $SCALA_HOME, $PATH pointed to it. For maven project, you should also update your pom.xml to depend on Scala -2.8-snapshot.

Below is a working snapshot that I was using new Scala plugin to write Scala plugin:

>>> Updated

Don't forget to download a fresh Scala 2.8.0-snapshot from scala-lang.org (you may want to track the progress of 2.8.0 once a week etc), and set SCALA_HOME, PATH environment to point to the installation of it.

======

During the weekend, I built a fresh Scala 2.8.0 snapshot. With  ticket #2163 fixed in 18 hours (great job, Scala team), I got a Scala plugin for NetBeans that works under Scala 2.8.0. I then re-built whole Erlang plugin (which is written in Scala), by fixing some minor incompatible issues, it works too.

So, at least, Scala 2.8 seems stable enough for writing a NetBeans plugin now.

Here's a snapshot showing some new features of 2.8.0

If you are eager to have some taste of Scala 2.8.0 with NetBeans plugin, and is patient to keep another NetBeans nightly version on your machine, then, wait for one more day from now for NetBeans hudson to build whole things, download the lasted NetBeans nightly version, and install "Scala Kit" module via "Update Center".

Don't forget to download a fresh Scala 2.8.0-snapshot (you may want to track the progress of 2.8.0) from scala-lang.org, and maintain your SCALA_HOME, and PATH environment for different Scala versions

BTW, if I have time, I'll begin to rewrite this plugin in Scala, on 2.8.0

I've used  Rats! parser generator heavily on Scala/Erlang plugin for NetBeans, but not write a plugin for Rats! itself.

So I spent my weekend days on a simple Rats! editor module, which implemented syntax highlighting. It's built on Scala.

Here's the snapshot:

It will be available in couple of days.

I'm pleased to announce Erlang plugin (ErlyBird) version 1 for NetBeans 6.7 is released.

NetBeans 6.7 RC3 or above is a requirement.

What's new:

• It's rewritten in Scala instead of Java
• More reliable instant rename
• Display extracted document information from source comment when doing auto-completion

To download, please go to:  https://sourceforge.net/project/showfiles.php?group_id=192439&package_id=226387

To install:

• Open NetBeans, go to "Tools" -> "Plugins", click on "Downloaded" tab title, click on "Add Plugins..." button, choose the directory where the Erlang plugin are unzipped, select all listed *.nbm files, following the instructions.
• Make sure your Erlang bin path is under OS environment PATH, you can also check/set your OTP path: From [Tools]->[Erlang Platform], fill in the full path of your 'erl.exe' or 'erl' file in "Interpreter", for instance: "C:/erl/bin/erl.exe". Or open the "Brows" dialog to locate the erlang installation.

<li>When you open/create an Erlang project first time, the OTP libs will be indexed. Take a coffee and wait, the indexing time varies from 10 to 30 minutes depending on your computer.

Feedback and bug reports are welcome.

>>> Updated on July 1, 2009

There are couple of reports on "Could not connect to compilation daemon.", mostly under Windows OS. It's a known issue, to resolve it, please check the following:

• SCALA_HOME is set to a fresh installed Scala runtime
• PATH includes $SCALA_HOME/bin
• If the build task still complain, try to run "fsc" or "scala" in a command window first

======

I'm pleased to announce the availability of Scala plugin version 1 for NetBeans 6.7

What's new:

• Use fsc instead of scalac as the project building compiler (If you've set SCALA_HOME, make sure $SCALA_HOME/bin is included in your PATH environment).
• Fixed setting breakpoint in closure statement.
• A basic import-fixer (Right click on source, then choose "Fix Imports").
• Code assistant for local vars and functions.
• Run/Debug single file.

To download, please go to:  https://sourceforge.net/project/showfiles.php?group_id=192439&package_id=256544&release_id=686747

For more information, please see  http://wiki.netbeans.org/Scala

Bug reports are welcome.

It works on NetBeans 6.7 RC1 or above.

For past years, I worked on AIOTrade project, which uses Machine Learning technology to find the structural data pattern.

Then, to find a better programming language for AIOTrade, I tried Erlang, Scala, and wrote IDEs for these candidate languages, I learnt a lot of Formal Language Processing.

Now, finally, I dove into Natural Language Processing, and have a good chance to composite the knowledge of Statistics Learning, Rule based Language Processing on massive structural/un-structural data under parallel computing environment.

The candidate programming language is Scala, I keep to improve the NetBeans support for Scala these days while reading the books/code of NLP. During the long weekend, I've got some improvements on it, here is a summary:

• Migrated to NetBeans' CSL
• Use fsc instead of scalac as the project building compiler
• Fixed setting breakpoint in closure statement
• A basic import-fixer
• Code assistant for local vars and functions
• Run/Debug single file

The plugins can be got for beta testing on NetBeans Developing Update Center (for NetBeans 6.7 beta+), and will be released around the date of official NetBeans 6.7.

Well, I've migrated the core math, indicator parts of AIOTrade to Scala. There are two noticeable advances for new AIOTrade.

The first is the look and feel of indicator writing. Since Scala is suitable for DSL, now the indicator looks like:

class ARBRIndicator extends ContIndicator {
    _sname = "AR/BR"
    _grids = Array(50f, 200f)
    
    val period = Factor("Period", 10)
    
    val up = Var[Float]("up")
    val dn = Var[Float]("dn")
    val bs = Var[Float]("bs")
    val ss = Var[Float]("ss")
    
    val ar = Var[Float]("AR", Plot.Line)
    val br = Var[Float]("BR", Plot.Line)
    
    def computeCont(begIdx:Int, size:Int) {
        for (i <- begIdx until size) {
            up(i) = H(i) - O(i)
            val up_sum_i = sum(i, up, period)
            
            dn(i) = O(i) - L(i)
            val dn_sum_i = sum(i, dn, period)
            
            ar(i) = up_sum_i / dn_sum_i * 100
            
            val bs_tmp = H(i) - C(i)
            bs(i) = Math.max(0, bs_tmp)
            val bs_sum_i = sum(i, bs, period)
            
            val ss_tmp = C(i) - L(i)
            ss(i) = Math.max(0, ss_tmp)
            val ss_sum_i = sum(i, ss, period)
            
            br(i) = bs_sum_i / ss_sum_i * 100
        }
    }   
}  

Vs Java one:

public class ARBRIndicator extends ContIndicator {
    _sname = "AR/BR";
    _grids = new Float[] {50f, 200f};
    
    Opt period = new DefaultOpt("Period", 10);
    
    Var<Float> up = new DefaultVar("up");
    Var<Float> dn = new DefaultVar("dn");
    Var<Float> bs = new DefaultVar("bs");
    Var<Float> ss = new DefaultVar("ss");
    
    Var<Float> ar = new DefaultVar("AR", Plot.Line);
    Var<Float> br = new DefaultVar("BR", Plot.Line);
    
    
    void computeCont(int begIdx) {
        for (int i = begIdx; i < _itemSize; i++) {
            
            up.set(i, H.get(i) - O.get(i));
            float up_sum_i = sum(i, up, period);
            
            dn.set(i, O.get(i) - L.get(i));
            float dn_sum_i = sum(i, dn, period);
            
            ar.set(i, up_sum_i / dn_sum_i * 100);
            
            float bs_tmp = H.get(i) - C.get(i);
            bs.set(i, Math.max(0, bs_tmp));
            float bs_sum_i = sum(i, bs, period);
            
            float ss_tmp = C.get(i) - L.get(i);
            ss.set(i, Math.max(0, ss_tmp));
            float ss_sum_i = sum(i, ss, period);
            
            br.set(i, bs_sum_i / ss_sum_i * 100);
        }
    }
}

The apply method from Scala simplifies setter/getter, which makes the formulator looks more natural.

The second is by implementing each indicator as Actor, the computing procedure of indicators can be easily distributed to multiple CPU cores, with so few code modification:

case object Compute
trait Computable extends Actor {

    // ----- actor's implementation
    def act = loop {
        react {
            case (Compute, fromTime:Long) => computeFrom(fromTime)
            case _ =>
        }
    }
    // ----- end of actor's implementation

    def computeFrom(time:Long) :Unit
    def computedTime :Long
    
    def factors :ArrayBuffer[Factor]
    def factors_=(factors:ArrayBuffer[Factor]) :Unit
    def factors_=(values:Array[Number]) :Unit
    
    def dispose :Unit
}

Computable is an Interface/Trait with sync method: computeFrom(Long), now by extending Computable with Actor, implementing a simple function act with react message processing block, all indicators (which extended Computable) can benefit from parallel computing now by calling:

indicator ! (Compute, fromTime)

instead of

indicator.computeFrom(time)

I've done some testing on my 4-core machine, which occupied about 380% CPU usage during running. This is, of course, a most easily implementation for parallel computing under JVM so far.

Another bonus is, I do not need to worry about concurrent calling on computeFrom(Long) now, since all calls will be triggered by Compute messages that are sent to actor's message queue, then be processed sequentially, there is no lock needed any more. The key:

Parallel computing actors + Sequential message driven computing per actor

Finally, after evaluated Erlang, Scala etc, wrote IDE tools for these languages, I began to migrate AIOTrade from Java to Scala, with the help of Scala for NetBeans of course.

The first step is re-writting basic modules to Scala smoothly, with little functional style code; Then, I'll re-design the APIs by Scala's advanced features, including function, trait, actors etc.

Since AIOTrade is a NetBeans suite project, with several NetBeans style modules integrated, I need a general purpose build.xml to get Scala based modules working. Here are build.xml and scala-build.xml, which can be used to write Scala based NetBeans platform modules.

First, you should create a regular NetBeans module project, then put/replace these ant files under your project's base director. You also need to create 2 NetBeans lib wrapper modules, one is for scala-library.jar, another is for scala-compile.jar as parts of your NetBeans suite project (or, check them out from AIOTrade's source repository)

build.xml:

<?xml version="1.0" encoding="UTF-8"?>
<project name="lib.math" default="netbeans" basedir=".">
    <import file="scala-build.xml"/>
</project>

scala-build.xml:

<?xml version="1.0" encoding="UTF-8"?>
<project name="scala-module" default="netbeans" basedir=".">
    <import file="nbproject/build-impl.xml"/>

    <target name="scala-taskdef" depends="init">
        <property name="scala.library" value="${cluster}/modules/ext/scala-library.jar"/>
        <property name="scala.compiler" value="${cluster}/modules/ext/scala-compiler.jar"/>
        <property name="scala.libs" value="${scala.library}:${scala.compiler}"/>

        <echo message="cluster: ${cluster}"/>
        <echo message="Compiling scala sources via ${scala.library}, ${scala.compiler}"/>

        <taskdef resource="scala/tools/ant/antlib.xml">
            <classpath>
                <pathelement location="${cluster}/modules/ext/scala-library.jar"/>
                <pathelement location="${cluster}/modules/ext/scala-compiler.jar"/>
            </classpath>
        </taskdef>
    </target>

    <property name="jar-excludes" value="**/*.java,**/*.form,**/package.html,**/doc-files/,**/*.scala"/>

    <target name="compile" depends="init,up-to-date,scala-taskdef" unless="is.jar.uptodate">
        <!-- javac's classpath should include scala.library and all these paths of "cp" -->
        <path id="javac.cp">
            <pathelement path="${scala.libs}"/>
            <pathelement path="${module.classpath}"/>
            <pathelement path="${cp.extra}"/>
        </path>
        <!-- scalac will check class dependencies deeply, so we can not rely on public package only which is refed by ${module.classpath} -->
        <path id="scalac.cp">
            <pathelement path="${scala.libs}"/>
            <pathelement path="${module.run.classpath}"/>
            <pathelement path="${cp.extra}"/>
        </path>
        <mkdir dir="${build.classes.dir}"/>
        <depend srcdir="${src.dir}" destdir="${build.classes.dir}" cache="build/depcache">
            <classpath refid="scalac.cp"/>
        </depend>
        <!-- scalac -->
        <scalac srcdir="${src.dir}" destdir="${build.classes.dir}" encoding="UTF-8" target="jvm-${javac.target}">
            <classpath refid="scalac.cp"/>
        </scalac>
        <!-- javac -->
        <nb-javac srcdir="${src.dir}" destdir="${build.classes.dir}" debug="${build.compiler.debug}" debuglevel="${build.compiler.debuglevel}" encoding="UTF-8"
                deprecation="${build.compiler.deprecation}" optimize="${build.compiler.optimize}" source="${javac.source}" target="${javac.target}" includeantruntime="false">
            <classpath refid="javac.cp"/>
            <compilerarg line="${javac.compilerargs}"/>
            <processorpath refid="processor.cp"/>
        </nb-javac>
        <!-- Sanity check: -->
        <pathconvert pathsep=":" property="class.files.in.src">
            <path>
                <fileset dir="${src.dir}">
                    <include name="**/*.class"/>
                </fileset>
            </path>
        </pathconvert>
        <fail>
            <condition>
                <not>
                    <equals arg1="${class.files.in.src}" arg2=""/>
                </not>
            </condition>
            You have stray *.class files in ${src.dir} which you must remove.
            Probably you failed to clean your sources before updating them.
        </fail>
        <!-- OK, continue: -->
        <copy todir="${build.classes.dir}">
            <fileset dir="${src.dir}" excludes="${jar-excludes}"/>
        </copy>
    </target>

    <target name="do-test-build" depends="projectized-common.do-test-build">
        <scalac srcdir="${test.unit.src.dir}" destdir="${build.test.unit.classes.dir}" excludes="${test.excludes}"
               encoding="UTF-8">
            <classpath refid="test.unit.cp"/>
        </scalac>
    </target>
</project>

BTW, the new source code of AIOTrade is controlled under Mercurial version control system on sourceforge.net, you can clone or brows the code at: AIOTrade source repository

Note: The whole project can not be successfully built yet.

I fixed AIOTrade timezone issue. And, the applet demo on aiotrade.com does not need to be a signed applet any more.

For recent experience, I'm re-considering the RIA platform choice. May applet be saved in the near future? or Flash, Javascript win? First, Java applet needs a whole new look and feel theme, which make it fitting into the web page, second, how can a Java applet access DOM tree easily?

But,

When Flash/Flex tries to take the place of Java applet, it will become another Java itself.

I've done some preliminary re-design on  http://aiotrade.com, put the AIOTrade applet on home page, it requires browser with Java plugin above JRE 1.5, the size is about 288k. You can input the symbol to get real-time quote charting. Symbol is same as Yahoo! finance. The quote data is also from Yahoo! finance.

There are still some bugs on it. I'll continue to improve it.

Here's the snapshot of whole home page after you input a symbol. You can remove the chart by press "Remove Quotes" button.

For RIA framework, there are choices of Flex/Flash, Applet/JavaFX etc. I saw a lot of real-time financial charts were written in Flex/Flash, all over the world wide web. Then, how about a Java applet financial charting after Java 6u10? I'd like a try, and here is the result:

• The whole size of a most featured AIOTrade is less than 300k
• On my Macbook, with Java 6u12 installed, the init time is about 1 sec.
• By properly arranging Swing layout, the applet can auto-scale to proper size fit in html page just like any element.

Below is a snapshot:

For IDE support, an indexer is very important for declaration-find, code-completion, re-factory etc. Indexer will monitor the standard lib and your project, analyzing the source/binary files, gather meta-information of classes, functions, global vars, usages, and store them in a search engine (It's a Lucene back-end in NetBeans). That means, you need to support project management and platform management first, so you know where's the class/source path of platform and your current opened projects.

ErlyBird implemented Erlang platform and project management in modules: erlang.platform and erlang.project, these code are derived from Tor's Ruby module, I've migrated them to CSL, but did not rewrote them in Scala. It's OK for integrating these Java written modules with Scala written erlang.editor module.

First, you should identify the class path type, you can register them in ErlangLanguage.scala as:

    /** @see org.netbeans.modules.erlang.platform.ErlangPlatformClassPathProvider and ModuleInstall */
    override
    def getLibraryPathIds = Collections.singleton(BOOT)

    override
    def getSourcePathIds = Collections.singleton(SOURCE)

object ErlangLanguage {
    val BOOT    = "erlang/classpath/boot"
    val COMPILE = "erlang/classpath/compile"
    val EXECUTE = "erlang/classpath/execute"
    val SOURCE  = "erlang/classpath/source"
}

Where, BOOT, thus getLibraryPathIds will point to Erlang OTP libs' path later in:
org.netbeans.modules.erlang.platform.ErlangPlatformClassPathProvider

Now, the indexer itself, ErlangIndex.scala which extends CSL's EmbeddingIndexer and implemented:

override protected def index(indexable:Indexable, parserResult:Result, context:Context) :Unit

with a factory:

class Factory extends EmbeddingIndexerFactory

Register it in ErlangLanguage.scala as:

    override
    def getIndexerFactory = new ErlangIndexer.Factory

Now, tell the CSL to monitor and index them, for Erlang OTP libs, this is done in ModuleInstall.java:

    @Override
    public void restored() {
        GlobalPathRegistry.getDefault().register(ErlangPlatformClassPathProvider.BOOT, new ClassPath[] { ErlangPlatformClassPathProvider.getBootClassPath() });
    }
    
    @Override
    public void uninstalled() {
        GlobalPathRegistry.getDefault().unregister(ErlangPlatformClassPathProvider.BOOT, new ClassPath[] { ErlangPlatformClassPathProvider.getBootClassPath() });
    }

All classpaths registered in GlobalPathRegistry will be monitored and indexed automatically. As "ModuleInstall.java" is also registered in module erlang.platform's manifest.mf as OpenIDE-Module-Install class, it will be automatically loaded and invoked when this module is installed/activated in NetBeans:

Manifest-Version: 1.0
OpenIDE-Module: org.netbeans.modules.erlang.platform
OpenIDE-Module-Layer: org/netbeans/modules/erlang/platform/resources/layer.xml
OpenIDE-Module-Localizing-Bundle: org/netbeans/modules/erlang/platform/Bundle.properties
OpenIDE-Module-Install: org/netbeans/modules/erlang/platform/ModuleInstall.class
OpenIDE-Module-Specification-Version: 0.18.0

For project's classpath, the "GlobalPathRegistry registering is at RubyProject.java (don't be confused by the name, it's a derivation from Ruby's code, I did not change it yet), as:

    private final class ProjectOpenedHookImpl extends ProjectOpenedHook {
        
        ProjectOpenedHookImpl() {}
        
        protected void projectOpened() {
            // register project's classpaths to GlobalPathRegistry
            ClassPathProviderImpl cpProvider = lookup.lookup(ClassPathProviderImpl.class);
            GlobalPathRegistry.getDefault().register(RubyProject.BOOT, cpProvider.getProjectClassPaths(RubyProject.BOOT));
            GlobalPathRegistry.getDefault().register(RubyProject.SOURCE, cpProvider.getProjectClassPaths(RubyProject.SOURCE));            
        }
        
        protected void projectClosed() {
            // unregister project's classpaths to GlobalPathRegistry
            ClassPathProviderImpl cpProvider = lookup.lookup(ClassPathProviderImpl.class);
            GlobalPathRegistry.getDefault().unregister(RubyProject.SOURCE, cpProvider.getProjectClassPaths(RubyProject.SOURCE));
        }
        
    }

So the project's source path will be registered when this project is opened, thus, trigger the index engine.

Now, when you first install Erlang plugin, IDE will index whole OTP libs once. And when an Erlang project is opened, the project's source files will be indexed too.

Then, you need to do a reverse job: search index, get the meta-data to composite Erlang symbols and AST items. This is done in ErlangIndex.scala, which has helper methods to search the index fields, and fetch back the stored signature string, extract it to symbol, and invoke the parsing manager to resolve the AST items. With the help of ErlangIndex.scala, you can search across the whole libs, find completion items, find declarations, find documents embedded in source comments, find usages.

As the first use of index feature, I've implemented the code-completion and go-to-declaration for global modules functions:

When user input "lists:", will invoke code-completion feature, since it's a remote function call, code-completion will search the index data, and find it's from OTP's "lists.erl", after parsing this file, we get all exported functions AstDfn, so, there are a lot of meta-information can be used now, I'll implement the documents tooltips later.

This will be the final one of the series of Erlang plug-in in Scala blogs, thanks for reading and feedback. I'll take a break during the weekend. From the next Monday, with the spring is coming, I'll join a new project, which is an ambitious financial information platform, I'll bring Scala, Lift, Erlang to it. BTW, congratulations to Lift 1.0!

Implementing Code-Completion is a bit complex, but you can got it work gradually. At the first step, you can implement Code-Completion for local vars/functions only, then, with the indexed supporting, you can add completion for remote functions.

You should define some kinds of completion proposal, which may show different behaviors when they are popped up and guard you followed steps. For example, a function proposal can auto-fill parameters, on the other side, a keyword proposal just complete itself.

The completion proposal classes are defined in  ErlangComplectionProposal.scala, which implemented CSL's interface CompletionProposal. you may notice that the function proposal is the most complex one, which should handle parameters information.

Then, you should implement CSL's interface CodeCompletionHandler, for Erlang, it's  ErlangCodeCompletion, where, the key method is:

    override
    def complete(context:CodeCompletionContext) :CodeCompletionResult = {
        this.caseSensitive = context.isCaseSensitive
        val pResult = context.getParserResult.asInstanceOf[ErlangParserResult]
        val lexOffset = context.getCaretOffset
        val prefix = context.getPrefix match {
            case null => ""
            case x => x
        }

        val kind = if (context.isPrefixMatch) QuerySupport.Kind.PREFIX else QuerySupport.Kind.EXACT
        val queryType = context.getQueryType

        val doc = LexUtil.document(pResult, true) match {
            case None => return CodeCompletionResult.NONE
            case Some(x) => x.asInstanceOf[BaseDocument]
        }

        val proposals = new ArrayList[CompletionProposal]
        val completionResult = new DefaultCompletionResult(proposals, false)

        // Read-lock due to Token hierarchy use
        doc.readLock
        try {
            val astOffset = LexUtil.astOffset(pResult, lexOffset)
            if (astOffset == -1) {
                return CodeCompletionResult.NONE
            }
            val root = pResult.rootScope match {
                case None => return CodeCompletionResult.NONE
                case Some(x) => x
            }
            val th = LexUtil.tokenHierarchy(pResult).get
            val fileObject = LexUtil.fileObject(pResult).get

            val request = new CompletionRequest
            request.completionResult = completionResult
            request.result = pResult
            request.lexOffset = lexOffset
            request.astOffset = astOffset
            request.index = ErlangIndex.get(pResult)
            request.doc = doc
            request.info = pResult
            request.prefix = prefix
            request.th = th
            request.kind = kind
            request.queryType = queryType
            request.fileObject = fileObject
            request.anchor = lexOffset - prefix.length
            request.root = root
            ErlangCodeCompletion.request = request
            
            val token = LexUtil.token(doc, lexOffset - 1) match {
                case None => return completionResult
                case Some(x) => x
            }

            token.id match {
                case ErlangTokenId.LineComment =>
                    // TODO - Complete symbols in comments?
                    return completionResult
                case ErlangTokenId.StringLiteral =>
                    //completeStrings(proposals, request)
                    return completionResult
                case _ =>
            }
            
            val ts = LexUtil.tokenSequence(th, lexOffset - 1) match {
                case None => return completionResult
                case Some(x) =>
                    x.move(lexOffset - 1)
                    if (!x.moveNext && !x.movePrevious) {
                        return completionResult
                    }
                    x
            }
 
            val closetToken = LexUtil.findPreviousNonWsNonComment(ts)

            if (root != null) {
                val sanitizedRange = pResult.sanitizedRange
                val offset = if (sanitizedRange != OffsetRange.NONE && sanitizedRange.containsInclusive(astOffset)) {
                    sanitizedRange.getStart
                } else astOffset

                val call = Call(null, null, false)
                findCall(root, ts, th, call, 0)
                val prefixBak = request.prefix
                call match {
                    case Call(null, _, _) =>
                    case Call(base, _, false) =>
                        // it's not a call, but may be candicate for module name, try to get modules and go-on
                        completeModules(base, proposals, request)
                    case Call(base, select, true) =>
                        if (select != null) {
                            request.prefix = call.select.text.toString
                        } else {
                            request.prefix = ""
                        }
                        completeModuleFunctions(call.base, proposals, request)
                        // Since is after a ":", we won't added other proposals, just return now whatever
                        return completionResult
                }
                request.prefix = prefixBak
                completeLocals(proposals, request)
            }

            completeKeywords(proposals, request)
        } finally {
            doc.readUnlock
        }

        completionResult
    }

For a Erlang function call, you should check the tokens surrounding the caret to get the call's base name and select first, which is done by a method findCall:

    private def findCall(rootScope:AstRootScope, ts:TokenSequence[TokenId], th:TokenHierarchy[_], call:Call, times:Int) :Unit = {
        assert(rootScope != null)
        val closest = LexUtil.findPreviousNonWsNonComment(ts)
        val idToken = closest.id match {
            case ErlangTokenId.Colon =>
                call.caretAfterColon = true
                // skip RParen if it's the previous
                if (ts.movePrevious) {
                    val prev = LexUtil.findPreviousNonWs(ts)
                    if (prev != null) {
                        prev.id match {
                            case ErlangTokenId.RParen   => LexUtil.skipPair(ts, ErlangTokenId.LParen,   ErlangTokenId.RParen,   true)
                            case ErlangTokenId.RBrace   => LexUtil.skipPair(ts, ErlangTokenId.LBrace,   ErlangTokenId.RBrace,   true)
                            case ErlangTokenId.RBracket => LexUtil.skipPair(ts, ErlangTokenId.LBracket, ErlangTokenId.RBracket, true)
                            case _ =>
                        }
                    }
                }
                LexUtil.findPrevIncluding(ts, LexUtil.CALL_IDs)
            case id if LexUtil.CALL_IDs.contains(id) => closest
            case _ => null
        }

        if (idToken != null) {
            times match {
                case 0 if call.caretAfterColon => call.base = idToken
                case 0 if ts.movePrevious => LexUtil.findPreviousNonWsNonComment(ts) match {
                        case null => call.base = idToken
                        case prev if prev.id == ErlangTokenId.Colon =>
                            call.caretAfterColon = true
                            call.select = idToken
                            findCall(rootScope, ts, th, call, times + 1)
                        case _ => call.base = idToken
                    }
                case _ => call.base = idToken
            }
        }
    }

    case class Call(var base:Token[TokenId], var select:Token[TokenId], var caretAfterColon:Boolean)

To complete a remote function call, you may need to visit outer modules, which needs an indexer, so as the first step, you can just ignore it, go straight to complete local vars/functions, or keywords:

    private def completeLocals(proposals:List[CompletionProposal], request:CompletionRequest) :Unit = {
        val prefix = request.prefix
        val kind = request.kind
        val pResult = request.result

        val root = request.root
        val closestScope = root.closestScope(request.th, request.astOffset) match {
            case None => return
            case Some(x) => x
        }
        val localVars = closestScope.visibleDfns(ElementKind.VARIABLE)
        localVars ++= closestScope.visibleDfns(ElementKind.PARAMETER)
        localVars.filter{v => filterKind(kind, prefix, v.name)}.foreach{v =>
            proposals.add(new PlainProposal(v, request.anchor))
        }

        val localFuns = closestScope.visibleDfns(ElementKind.METHOD)
        localFuns.filter{f => filterKind(kind, prefix, f.name)}.foreach{f =>
            proposals.add(new FunctionProposal(f, request.anchor))
        }
    }

    private def completeKeywords(proposals:List[CompletionProposal], request:CompletionRequest) :Unit = {
        val prefix = request.prefix
        val itr = LexerErlang.ERLANG_KEYWORDS.iterator
        while (itr.hasNext) {
            val keyword = itr.next
            if (startsWith(keyword, prefix)) {
                proposals.add(new KeywordProposal(keyword, null, request.anchor))
            }
        }
    }

There is a function def visibleDfns(kind: ElementKind): ArrayBuffer[AstDfn] in AstRootScope.scala, if you've put definition items properly in scopes, it should handle the visibility automatically.

Now, register it in  ErlangLanguage.scala:

    override
    def getCompletionHandler = new ErlangCodeCompletion

As usual, run it, you got:

The local functions and vars are proposed plus the keywords. BTW, I've fixed this feature for Scala plug-in.

Case class in Scala is a neat feature, it automatically generates apply, extract and equals functions. But I encountered a strange "equals" behavior today, which wasted me 2 hours to find why.

Define a simple case class "Symbol", then a "VarSymbol" which extends "Symbol" with one more field "name:String". We know you can compare the equality of instances of "VarSymbol" by this "name", Scala automatically gets it right. And of course, a direct instance from "Symbol()" should not equal to any "VarSymbol", since it lacks "name" field, that's right.

case class Symbol()
case class VarSymbol(name:String) extends Symbol

Then, I need a singleton object "NoSymbol", I defined it as:

object NoSymbol extends Symbol

I of course thought this "NoSymbol" should not equal any "VarSymbol" instance, or,

NoSymbol == VarSymbol("I'm var")

Should return false. But life is not so straightforward, it returns true in real life !!!

I finally got my code working, by adding a "case" before "object NoSymbol extends Symbol". This "NoSymbol" won't equal any "VarSymbol" now, I'm grad things finally go back in track.

I don't know if it's a Scala bug, or, that's what Scala thinks it should be: all "Symbol" and it's inherited instances should equal "NoSymbol" object. Anyway, here's the whole code for test:

case class Symbol()
case class VarSymbol(name:String) extends Symbol

object NoSymbol extends Symbol
case object CasedNoSymbol extends Symbol

object TestCaseObject {
    
    def test = {
        val noSym = NoSymbol
        val caseNoSym = CasedNoSymbol
        val varSym = VarSymbol("I'm var")

        if (noSym == varSym) println("NoSym equals varSym !") 
        else println("NoSym doesn't equal varSym")
        
        if (caseNoSym == varSym) println("CaseNoSym equals varSym !") 
        else println("CaseNoSym doesn't equal varSym")
    }
    
}

Run TestCaseObject.test, I got:

NoSym equals varSym !
CaseNoSym doesn't equal varSym

NetBeans 6.7M2 is going to be public available soon, the new look&feel on Mac OS is very awesome. BTW, -Dapple.awt.graphics.UseQuartz=true is a must option for best font-rendering on my macbook. I added it to netbeans.conf as -J-Dapple.awt.graphics.UseQuartz=true. Here is a snapshot of my current screen when working on Erlang plugin in Scala:

Click on the picture to enlarge it:

There is a Java class which takes type parameter T, T is any type:

package dcaoyuan.test;

import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;

public class AList<T> implements Iterator<T> {

   List<T> a = new ArrayList<T>();

   {
       a.add((T) "string");
   }

   Iterator<T> itr = a.iterator();

   public boolean hasNext() {
       return itr.hasNext();
   }

   public T next() {
       return itr.next();
   }

   public void remove() {
       throw new UnsupportedOperationException("Not supported yet.");
   }
}

Which implemented java.util.Iterator with type parametered T next(). Notice here next() will return T

The above code simulates a case, that an existed Java lib may return an instance that has type parameter erased, on run-time.

Now let's try to call it in Scala:

package dcaoyuan.test

object Main {

   def main(args: Array[String]) :Unit = {
       testAList_work
       testAList_unwork
   }

   def testAList_work :Unit = {
       val a :AList[_] = new AList
       // a is inferred as AList[Any]
       while (a.hasNext) {
           val e = a.next
           println(e)
       }
   }

   def testAList_unwork :Unit = {
       val a = new AList
       // a is inferred as AList[Nothing]
       while (a.hasNext) {
           // now a.next is Nothing, can not cast to "val e", which should at least be Any
           val e = a.next // will throw java.lang.ClassCastException
           println(e)
       }
   }
}

I wrote two functions: "testAList_work" and "testAList_unwork"

The issue is in testAList_unwork, as a simple "val a = new AList", "a" will be inferred by scalac as AList[Nothing], so, "a.next" will return an instance of type Nothing, then, "val e = a.next" will throw java.lang.ClassCastException

My question is, should this corner case be checked by scalac as an error or warning? So I do not need to dig the clause when it happens on run-time.

The progress of rewriting Erlang plugin for NetBeans in Scala has reached a phase, that the Editor itself works smooth and better than ErlyBird now, the next step is to integrate an Erlang project management and index the modules/functions of OTP/project to provide smart auto-completion.

Now Scala has been proved that it can be integrated into an existed large Java based framework (NetBeans IDE here) without no much problems, I'll begin to rewrite Scala plugin in Scala soon.

Although in most cases, Scala can call existed Java code or classes smoothly, there are still some corner cases. I'll record these corner cases in blogs. Here's the first one which I also posted on scala-user mailing-list, but have not yet got final answer.

Let's begin with a detailed example:

There is a Java interface A:

public interface A<T extends String> {
   void run(T t);
}

Which has a type parameter <T extends String> and abstract method run(T t)

Then a Java abstract class B extended A. But, B, as it, omitted type parameter from A. This is unsafe but valid in Java:

public abstract class B implements A {
   public String me() {
       return "I'm B";
   }
}

Assume above classes A and B have been compiled under javac, and packed in a jar library, and I can not patch it anymore. Now I need to write a class S in Scala which should extend B:

class S extends B {
   override
   def run[T <: String](t:T) = {println(t)}
}

scalac will complain as:

/Users/dcaoyuan/NetBeansProjects/ScalaTestCase/src/S.scala:1: error:
class S needs to be abstract, since method run in trait A of type
(T)Unit is not defined
class S extends B {
/Users/dcaoyuan/NetBeansProjects/ScalaTestCase/src/S.scala:3: error:
method run overrides nothing
   def run[T <: String](t:T) = {println(t)}

I than tried "forSome" type:

class S extends B {
   override
   def run(t:T forSome {type T <: String}) = {println(t)}
}

The code still did not work.

It seems that, since B omitted A's type parameter T, I have no way to get what is T, and can not successfully implement "run(T t)" method.

I also tried other forms of "forSome" usages, and always failed.

But I think Scala can always be saved with mixed Java/Scala code in such corner case, that's what I believed. So, I thought about it later when my brain was spare, and finally got a solution:

I wrote another Java abstract class B1 which extends B and pretended to have implemented "run(T t)", but actually called another new abstract method "runImpl(String t)"

public abstract class B1 extends B {

    public void run(String t) {
        runImpl(t);
    }

    public abstract void runImpl(String t);
}

Now I can let Scala S extends B1 and implement "runImpl(String t)" instead of extending B and implementing "run(T t)".

class S extends B1 {
    override
    def runImpl(t:String) = {println(t)}
}

Yes, scalac won't complain about "runImpl(t:String)" at all, and I got S successfully extends B by bridge class B1.

But I still hope scalac can resolve it directly, with a warning message instead of failing to compile it.

It seems I'm the only one who is committing to hg.netbeans.org/main/contrib these days. Anyway.

Implementing Instant Rename and Go to Declaration is not difficult with AstDfn/AstRef/AstScope. Here is the code:

• Instant Rename is implemented by ErlangInstantRenamer.scala
• Go to Declaration is implemented by ErlangDeclarationFinder.scala

And register them in :

    override
    def getInstantRenamer = new ErlangInstantRenamer

    override
    def getDeclarationFinder = new ErlangDeclarationFinder

Only local function definitions can be traveled to at this time. To jump to remote function definitions, I have to implement an indexer first.

Now let's go on the complex part: Pretty Formatting and Pair Matching. I say they are complex, not because these features are much heavier on language's semantic complex. Implementing these features mostly deals with lexer tokens. But it's a bit brain-dried work to across forward/backward in the token stream to get the pair matching and pretty formatting working as you expected.

Because of the complex, I won't describe the details of how to implement them for Erlang, I just put the links to these source code:

• Pretty Formatting is implemented by ErlangFormatter.scala
• Pair Matching is implemented by ErlangKeystrokeHandler.scala

And registered them in ErlangLanguage.scala as:

    override
    def getKeystrokeHandler = new ErlangKeystrokeHandler

    override
    def hasFormatter =  true

    override
    def getFormatter = new ErlangFormatter

With these feature implemented, the new plugin can automatically complete/match braces and pair, indent properly when you hit BREAK, input a "end" etc.

BTW, the navigator window was improved during these days, it can now properly show the arity/args of each functions. It's done by improved AstNodeVisitor.scala and AstDfn.scala

During the AST node visiting (AstNodeVisitor.scala), I've gathered a lot of variable definitions and their references, we can now try to get editor to mark these occurrences. There are preliminary functions in AstScope.scala, such as findOccurrences(AstItem). You can override AstDfn#isReferredBy(AstRef) and AstRef#isOccurence(AstRef) to get accurate reference relations. As the first step, I just simply judge the reference relation by the equation of names.

We'll extends org.netbeans.modules.csl.api.OccurrencesFinder and implement:

run(pResult:ErlangParserResult, event:SchedulerEvent) :Unit

First, we try to get the AstItem which is at the caretPosition by rootScope.findItemAt(th, caretPosition), and verify if it's a valid token.

Then, by calling rootScope.findOccurrences(item), we collect all occurrences of this item, and put a ColoringAttributes.MARK_OCCURRENCES with its OffsetRange.

The code of ErlangOccurrencesFinder.scala:

/*
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER.
 *
 * Copyright 1997-2007 Sun Microsystems, Inc. All rights reserved.
 *
 * The contents of this file are subject to the terms of either the GNU
 * General Public License Version 2 only ("GPL") or the Common
 * Development and Distribution License("CDDL") (collectively, the
 * "License"). You may not use this file except in compliance with the
 * License. You can obtain a copy of the License at
 * http://www.netbeans.org/cddl-gplv2.html
 * or nbbuild/licenses/CDDL-GPL-2-CP. See the License for the
 * specific language governing permissions and limitations under the
 * License.  When distributing the software, include this License Header
 * Notice in each file and include the License file at
 * nbbuild/licenses/CDDL-GPL-2-CP.  Sun designates this
 * particular file as subject to the "Classpath" exception as provided
 * by Sun in the GPL Version 2 section of the License file that
 * accompanied this code. If applicable, add the following below the
 * License Header, with the fields enclosed by brackets [] replaced by
 * your own identifying information:
 * "Portions Copyrighted [year] [name of copyright owner]"
 *
 * If you wish your version of this file to be governed by only the CDDL
 * or only the GPL Version 2, indicate your decision by adding
 * "[Contributor] elects to include this software in this distribution
 * under the [CDDL or GPL Version 2] license." If you do not indicate a
 * single choice of license, a recipient has the option to distribute
 * your version of this file under either the CDDL, the GPL Version 2 or
 * to extend the choice of license to its licensees as provided above.
 * However, if you add GPL Version 2 code and therefore, elected the GPL
 * Version 2 license, then the option applies only if the new code is
 * made subject to such option by the copyright holder.
 *
 * Contributor(s):
 *
 * Portions Copyrighted 2009 Sun Microsystems, Inc.
 */
package org.netbeans.modules.erlang.editor

import _root_.java.util.{HashMap,List,Map}
import javax.swing.text.Document
import org.netbeans.api.lexer.{Token,TokenId,TokenHierarchy}
import _root_.org.netbeans.modules.csl.api.{ColoringAttributes,OccurrencesFinder,OffsetRange}
import org.netbeans.modules.parsing.spi.Parser
import org.netbeans.modules.parsing.spi.{Scheduler,SchedulerEvent}
import org.netbeans.modules.erlang.editor.ast.{AstDfn,AstItem,AstRef,AstRootScope}
import org.netbeans.modules.erlang.editor.lexer.{ErlangTokenId,LexUtil}
import org.openide.filesystems.FileObject

/**
 *
 * @author Caoyuan Deng
 */
class ErlangOccurrencesFinder extends OccurrencesFinder[ErlangParserResult] {

    protected var cancelled = false
    private var caretPosition = 0
    private var occurrences :Map[OffsetRange, ColoringAttributes] = _
    private var file :FileObject = _

    protected def isCancelled = synchronized {cancelled}

    protected def resume :Unit = synchronized {cancelled = false}

    override
    def getPriority = 0

    override
    def getSchedulerClass = Scheduler.CURSOR_SENSITIVE_TASK_SCHEDULER

    override
    def getOccurrences :Map[OffsetRange, ColoringAttributes] = occurrences

    override
    def cancel :Unit = synchronized {cancelled = true}

    override
    def setCaretPosition(position:Int) :Unit = {this.caretPosition = position}

    def run(pResult:ErlangParserResult, event:SchedulerEvent) :Unit = {
        resume
        if (pResult == null || isCancelled) {
            return
        }

        val currentFile = pResult.getSnapshot.getSource.getFileObject
        if (currentFile != file) {
            // Ensure that we don't reuse results from a different file
            occurrences = null
            file = currentFile
        }

        for (rootScope <- pResult.rootScope;
             th <- LexUtil.tokenHierarchy(pResult);
             doc <- LexUtil.document(pResult, true);
             // * we'll find item by offset of item's idToken, so, use caretPosition directly
             item <- rootScope.findItemAt(th, caretPosition);
             idToken <- item.idToken
        ) {
            var highlights = new HashMap[OffsetRange, ColoringAttributes](100)

            val astOffset = LexUtil.astOffset(pResult, caretPosition)
            if (astOffset == -1) {
                return
            }

            // * When we sanitize the line around the caret, occurrences
            // * highlighting can get really ugly
            val blankRange = pResult.sanitizedRange

            if (blankRange.containsInclusive(astOffset)) {
                return
            }

            // * test if document was just closed?
            LexUtil.document(pResult, true) match {
                case None => return
                case _ =>
            }

            try {
                doc.readLock
                val length = doc.getLength
                val astRange = LexUtil.rangeOfToken(th.asInstanceOf[TokenHierarchy[TokenId]], idToken.asInstanceOf[Token[TokenId]])
                val lexRange = LexUtil.lexerOffsets(pResult, astRange)
                var lexStartPos = lexRange.getStart
                var lexEndPos = lexRange.getEnd

                // If the buffer was just modified where a lot of text was deleted,
                // the parse tree positions could be pointing outside the valid range
                if (lexStartPos > length) {
                    lexStartPos = length
                }
                if (lexEndPos > length) {
                    lexEndPos = length
                }

                LexUtil.token(doc, caretPosition) match {
                    case None => return
                    case token => // valid token, go on
                }
            } finally {
                doc.readUnlock
            }

            val _occurrences = rootScope.findOccurrences(item)
            for (_item <- _occurrences;
                 _idToken <- _item.idToken
            ) {
                highlights.put(LexUtil.rangeOfToken(th.asInstanceOf[TokenHierarchy[TokenId]],
                                                    _idToken.asInstanceOf[Token[TokenId]]),
                               ColoringAttributes.MARK_OCCURRENCES)
            }

            if (isCancelled) {
                return
            }

            if (highlights.size > 0) {
                val translated = new HashMap[OffsetRange, ColoringAttributes](2 * highlights.size)
                val entries = highlights.entrySet.iterator
                while (entries.hasNext) {
                    val entry = entries.next
                    LexUtil.lexerOffsets(pResult, entry.getKey) match {
                        case OffsetRange.NONE =>
                        case range => translated.put(range, entry.getValue)
                    }
                }

                highlights = translated

                this.occurrences = highlights
            } else {
                this.occurrences = null
            }
        }
    }
}

Finally, again, register it in ErlangLanguage.scala

override def hasOccurrencesFinder = true
override def getOccurrencesFinder = new ErlangOccurrencesFinder

And add mark-occurrences effect setting in fontsColors.xml:

<fontcolor name="mark-occurrences" bgColor="ECEBA3"/>

That's all. Run it, you got:

Var LogPid's definition and references were highlighted now.

The correctness of occurrences marking depends on the correctness of scopes of variables when you visit AST node, I did not finish all of such work yet, but, enabling occurrences marking feature is very helpful for further work.

With more detailed AstNodeVisitor.scala, I got the semantic information of function calls, variable definitions and references etc. It's time to implement CSL's SemanticAnalyzer, which is the entrance of semantic highlighting.

I then encountered a Scala's corner case issue :-(

SemanticAnalyzer.java is a sub-class of ParserResultTask:

package org.netbeans.modules.parsing.spi;
public abstract class ParserResultTask<T extends Parser.Result> extends SchedulerTask {
    public abstract void run (T result, SchedulerEvent event);
    public abstract int getPriority ();
}

ParserResultTask's signature in class file is:

<<TLorg/netbeans/modules/parsing/spi/Parser$Result;>Lorg/netbeans/modules/parsing/spi/SchedulerTask;>

run's method signature:

<(TT;Lorg/netbeans/modules/parsing/spi/SchedulerEvent;)V>

ErlangSemanticAnalyzer extended SemanticAnalyzer, so I should implement:

void run (T result, SchedulerEvent event);

which carries type parameter T from ParserResultTask<T extends Parser.Result>. But unfortunately, SemanticAnalyzer extends ParserResultTask as:

public abstract class SemanticAnalyzer extends ParserResultTask;

That is, SemanticAnalyzer erased type parameter of its parent. It's valid in Java. But in Scala, I can not successfully extend SemanticAnalyzer anymore, since when I tried to implement "void run (T result, SchedulerEvent event)", the scalac always complained that I did not override this method with type T. It seems scalac checks type parameter deeply, and needs sub-class to always explicitly extend parent class with type parameter. Since SemanticAnalyzer has erased type parameter, even I tried to implement "run" as:

override def run[T <: Parser.Result](result:T, event:SchedulerEvent) :Unit 

scalac still complained about it.

I have no idea of how to bypass this issue. I then patched SemanticAnalyzer.java, let it carries same type parameter as its parent:

public abstract class SemanticAnalyzer<T extends Parser.Result> extends ParserResultTask<T> 

Now everything works. My final code is:

/*
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER.
 *
 * Copyright 1997-2008 Sun Microsystems, Inc. All rights reserved.
 *
 * The contents of this file are subject to the terms of either the GNU
 * General Public License Version 2 only ("GPL") or the Common
 * Development and Distribution License("CDDL") (collectively, the
 * "License"). You may not use this file except in compliance with the
 * License. You can obtain a copy of the License at
 * http://www.netbeans.org/cddl-gplv2.html
 * or nbbuild/licenses/CDDL-GPL-2-CP. See the License for the
 * specific language governing permissions and limitations under the
 * License.  When distributing the software, include this License Header
 * Notice in each file and include the License file at
 * nbbuild/licenses/CDDL-GPL-2-CP.  Sun designates this
 * particular file as subject to the "Classpath" exception as provided
 * by Sun in the GPL Version 2 section of the License file that
 * accompanied this code. If applicable, add the following below the
 * License Header, with the fields enclosed by brackets [] replaced by
 * your own identifying information:
 * "Portions Copyrighted [year] [name of copyright owner]"
 *
 * If you wish your version of this file to be governed by only the CDDL
 * or only the GPL Version 2, indicate your decision by adding
 * "[Contributor] elects to include this software in this distribution
 * under the [CDDL or GPL Version 2] license." If you do not indicate a
 * single choice of license, a recipient has the option to distribute
 * your version of this file under either the CDDL, the GPL Version 2 or
 * to extend the choice of license to its licensees as provided above.
 * However, if you add GPL Version 2 code and therefore, elected the GPL
 * Version 2 license, then the option applies only if the new code is
 * made subject to such option by the copyright holder.
 *
 * Contributor(s):
 *
 * Portions Copyrighted 2009 Sun Microsystems, Inc.
 */
package org.netbeans.modules.erlang.editor

import _root_.java.util.{HashMap,HashSet,Map,Set}
import javax.swing.text.Document
import org.netbeans.api.lexer.Token
import org.netbeans.api.lexer.TokenHierarchy
import org.netbeans.api.lexer.TokenId
import org.netbeans.modules.csl.api.{ColoringAttributes,ElementKind,OffsetRange,SemanticAnalyzer}
import org.netbeans.modules.parsing.spi.Parser
import org.netbeans.modules.parsing.spi.{Scheduler,SchedulerEvent,ParserResultTask}
import org.netbeans.modules.erlang.editor.ast.{AstDfn,AstItem,AstRef,AstRootScope}
import org.netbeans.modules.erlang.editor.lexer.LexUtil
import org.netbeans.modules.erlang.editor.lexer.ErlangTokenId

/**
 *
 * @author Caoyuan Deng
 */
class ErlangSemanticAnalyzer extends SemanticAnalyzer[ErlangParserResult] {

    private var cancelled = false
    private var semanticHighlights :Map[OffsetRange, Set[ColoringAttributes]] = _

    protected def isCancelled :Boolean = synchronized {cancelled}

    protected def resume :Unit = synchronized {cancelled = false}

    override
    def getHighlights :Map[OffsetRange, Set[ColoringAttributes]] = semanticHighlights

    override
    def getPriority = 0

    override
    def getSchedulerClass = Scheduler.EDITOR_SENSITIVE_TASK_SCHEDULER
    
    override
    def cancel :Unit = {cancelled = true}

    @throws(classOf[Exception])
    override
    def run(pResult:ErlangParserResult, event:SchedulerEvent) :Unit = {
        resume
        semanticHighlights = null

        if (pResult == null || isCancelled) {
            return
        }

        for (rootScope <- pResult.rootScope;
             th <- LexUtil.tokenHierarchy(pResult);
             doc <- LexUtil.document(pResult, true)
        ) {
            var highlights = new HashMap[OffsetRange, Set[ColoringAttributes]](100)
            visitItems(th.asInstanceOf[TokenHierarchy[TokenId]], rootScope, highlights)

            this.semanticHighlights = if (highlights.size > 0) highlights else null
        }
    }

    private def visitItems(th:TokenHierarchy[TokenId], rootScope:AstRootScope, highlights:Map[OffsetRange, Set[ColoringAttributes]]) :Unit = {
        import ElementKind._
        for (item <- rootScope.idTokenToItem(th).values;
             hiToken <- item.idToken
        ) {
            
            val hiRange = LexUtil.rangeOfToken(th, hiToken.asInstanceOf[Token[TokenId]])
            item match {
                case dfn:AstDfn => dfn.getKind match {
                        case MODULE =>
                            highlights.put(hiRange, ColoringAttributes.CLASS_SET)
                        case CLASS =>
                            highlights.put(hiRange, ColoringAttributes.CLASS_SET)
                        case ATTRIBUTE =>
                            highlights.put(hiRange, ColoringAttributes.STATIC_SET)
                        case METHOD =>
                            highlights.put(hiRange, ColoringAttributes.METHOD_SET)
                        case PARAMETER =>
                            highlights.put(hiRange, ColoringAttributes.PARAMETER_SET)
                        case _ =>
                    }
                case ref:AstRef => ref.getKind match {
                        case METHOD =>
                            highlights.put(hiRange, ColoringAttributes.FIELD_SET)
                        case PARAMETER =>
                            highlights.put(hiRange, ColoringAttributes.PARAMETER_SET)
                        case _ =>
                    }
            }
        }
    }
}

In ErlangSemanticeAnalyzer, what you need to do is traversing all AstDfn and AstRef instances, and give them a proper ColoringAttributes set. I marked all functions as METHOD_SET(mod-method), attributes as STATIC_SET(mod-static), and function call names as FIELD_SET(mod-field), parameters as PARAMETER_SET(mod-parameter), the names in parenthesis are the corresponding names that in fontColors.xml setting file, you can define the highlighting effects in this xml file.

Again, don't forget to register it in ErlangLanguage.scala:

override def getSemanticAnalyzer = new ErlangSemanticAnalyzer

Run it, and I got highlighted source code as:

The function declaration names are underline, parameters are bold, function calls are bold too.

I'll gather even more detailed semantic information later, so I can identity the unused variable definitions and var references without definitions etc.

During the weekend, I've done some preliminary error recover work for Erlang's rats! definition. Now I'll go on some features based on analysis on AST tree. As the simplest step, we'll visit/analyze AST tree to get the structure information, use them for Navigator window and code folding.

First, we need to record the Structure information in some way. Each language has different AST tree and may be generated by varies tools, the AST element should be wrapped in some more generic classes to get integrated into CSL framework. There is an interface "org.netbeans.modules.csl.api.ElementKind", which is used for this purpose. But it's not enough, we need some facilities to not only wrap AST element, and also help us to identify the element as a definition or reference, in which visible scope etc.

I wrote a couple of these facilities when I wrote Erlang/Fortress/Scala plug-ins, these facilities can be repeatedly used for other languages by minor modifying. They are AstItem, AstDfn, AstRef, AstScope, AstRootScope. For Erlang plugin, you can find them under director: erlang.editor/src/org/netbeans/modules/erlang/editor/ast

AstDfn.scala is used to store a definition, such as definitions of class, module, method, variable, attribute etc. AstRef.scala is used to store reference that refers to definition, for example, the usages of a class, a variable, a method call etc.

There should be some lexer level utilities too, to help you get the corresponding tokens when you visit AST tree. It's LexUtil.scala, which will be also heavy used for code-folding, indentation ... features.

With above facilities, we can visit an AST tree now, I'll do the simplest task first: get all functions/attribues name and bounds tokens.

AstNodeVisitor.scala

/*
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER.
 * 
 * Copyright 2008 Sun Microsystems, Inc. All rights reserved.
 * 
 * The contents of this file are subject to the terms of either the GNU
 * General Public License Version 2 only ("GPL") or the Common
 * Development and Distribution License("CDDL") (collectively, the
 * "License"). You may not use this file except in compliance with the
 * License. You can obtain a copy of the License at
 * http://www.netbeans.org/cddl-gplv2.html
 * or nbbuild/licenses/CDDL-GPL-2-CP. See the License for the
 * specific language governing permissions and limitations under the
 * License.  When distributing the software, include this License Header
 * Notice in each file and include the License file at
 * nbbuild/licenses/CDDL-GPL-2-CP.  Sun designates this
 * particular file as subject to the "Classpath" exception as provided
 * by Sun in the GPL Version 2 section of the License file that
 * accompanied this code. If applicable, add the following below the
 * License Header, with the fields enclosed by brackets [] replaced by
 * your own identifying information:
 * "Portions Copyrighted [year] [name of copyright owner]"
 * 
 * If you wish your version of this file to be governed by only the CDDL
 * or only the GPL Version 2, indicate your decision by adding
 * "[Contributor] elects to include this software in this distribution
 * under the [CDDL or GPL Version 2] license." If you do not indicate a
 * single choice of license, a recipient has the option to distribute
 * your version of this file under either the CDDL, the GPL Version 2 or
 * to extend the choice of license to its licensees as provided above.
 * However, if you add GPL Version 2 code and therefore, elected the GPL
 * Version 2 license, then the option applies only if the new code is
 * made subject to such option by the copyright holder.
 * 
 * Contributor(s):
 * 
 * Portions Copyrighted 2009 Sun Microsystems, Inc.
 */
package org.netbeans.modules.erlang.editor.node

import org.netbeans.api.lexer.{Token, TokenId, TokenHierarchy, TokenSequence}
import org.netbeans.modules.csl.api.ElementKind

import org.netbeans.modules.erlang.editor.ast.{AstDfn, AstItem, AstRef, AstRootScope, AstScope, AstVisitor}
import org.netbeans.modules.erlang.editor.lexer.ErlangTokenId._
import org.netbeans.modules.erlang.editor.lexer.{ErlangTokenId, LexUtil}
import org.openide.filesystems.FileObject

import scala.collection.mutable.ArrayBuffer

import xtc.tree.GNode
import xtc.tree.Node
import xtc.util.Pair

/**
 *
 * @author Caoyuan Deng
 */
class AstNodeVisitor(rootNode:Node, th:TokenHierarchy[_], fo:FileObject) extends AstVisitor(rootNode, th) {

    def visitS(that:GNode) = {
        val formNodes = that.getList(0).iterator
        while(formNodes.hasNext) {
            visitForm(formNodes.next)
        }
    }

    def visitForm(that:GNode) = {
        enter(that)

        val scope = new AstScope(boundsTokens(that))
        rootScope.addScope(scope)

        scopes.push(scope)
        visitNodeOnly(that.getGeneric(0))
        
        exit(that)
        scopes.pop
    }


    def visitAttribute(that:GNode) = {
        that.get(0) match {
            case atomId:GNode =>
                val attr = new AstDfn(that, idToken(idNode(atomId)), scopes.top, ElementKind.ATTRIBUTE, fo)
                rootScope.addDfn(attr)
        }
    }

    def visitFunction(that:GNode) = {
        visitFunctionClauses(that.getGeneric(0))
    }

    def visitFunctionClauses(that:GNode) = {
        val fstClauseNode = that.getGeneric(0)
        visitFunctionClause(fstClauseNode)
    }

    def visitFunctionClause(that:GNode) = {
        val id = idNode(that.getGeneric(0))
        val fun = new AstDfn(that, idToken(id), scopes.top, ElementKind.METHOD, fo)
        rootScope.addDfn(fun)
    }

    def visitRule(that:GNode) = {

    }
}

As you can see, I just visit function/attribute related AST nodes, and gather all function/attribute declarations (or, definitions). My AST tree is generated by rats!, so the code looks like above. If you are using other parsers, I assume you know of course how to travel it.

AstNodeVisitor extends AstVisitor, I wrote some helper methods in AstVisitor.scala, for example, getting the bounds tokens for a definition/scope.

Now, you need to put the AST visiting task to ErlangParser.scala, so, it will be called when parsing finished, and you'll get an AST tree. The code is something like:

    private def analyze(context:Context) :Unit = {
        val doc = LexUtil.document(context.snapshot, false)

        // * we need TokenHierarchy to do anaylzing task
        for (root <- context.root;
             th <- LexUtil.tokenHierarchy(context.snapshot)) {
            // * Due to Token hierarchy will be used in analyzing, should do it in an Read-lock atomic task
            for (x <- doc) {x.readLock}
            try {
                val visitor = new AstNodeVisitor(root, th, context.fo)
                visitor.visit(root)
                context.rootScope = Some(visitor.rootScope)
            } catch {
                case ex:Throwable => ex.printStackTrace
            } finally {
                for (x <- doc) {x.readUnlock}
            }
        }
    }

The rootScope will be carried in ErlangParserResult.scala.

With the visitor's rootScope, we can implement the navigator window and code folding now. What you need is to implement an org.netbeans.modules.csl.api.StructureScanner. My implementation is: ErlangStructureAnalyzer.scala

/*
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER.
 *
 * Copyright 1997-2007 Sun Microsystems, Inc. All rights reserved.
 *
 * The contents of this file are subject to the terms of either the GNU
 * General Public License Version 2 only ("GPL") or the Common
 * Development and Distribution License("CDDL") (collectively, the
 * "License"). You may not use this file except in compliance with the
 * License. You can obtain a copy of the License at
 * http://www.netbeans.org/cddl-gplv2.html
 * or nbbuild/licenses/CDDL-GPL-2-CP. See the License for the
 * specific language governing permissions and limitations under the
 * License.  When distributing the software, include this License Header
 * Notice in each file and include the License file at
 * nbbuild/licenses/CDDL-GPL-2-CP.  Sun designates this
 * particular file as subject to the "Classpath" exception as provided
 * by Sun in the GPL Version 2 section of the License file that
 * accompanied this code. If applicable, add the following below the
 * License Header, with the fields enclosed by brackets [] replaced by
 * your own identifying information:
 * "Portions Copyrighted [year] [name of copyright owner]"
 *
 * If you wish your version of this file to be governed by only the CDDL
 * or only the GPL Version 2, indicate your decision by adding
 * "[Contributor] elects to include this software in this distribution
 * under the [CDDL or GPL Version 2] license." If you do not indicate a
 * single choice of license, a recipient has the option to distribute
 * your version of this file under either the CDDL, the GPL Version 2 or
 * to extend the choice of license to its licensees as provided above.
 * However, if you add GPL Version 2 code and therefore, elected the GPL
 * Version 2 license, then the option applies only if the new code is
 * made subject to such option by the copyright holder.
 *
 * Contributor(s):
 *
 * Portions Copyrighted 2009 Sun Microsystems, Inc.
 */
package org.netbeans.modules.erlang.editor;

import _root_.java.util.{ArrayList,Collections,HashMap,List,Map,Set,Stack}
import javax.swing.ImageIcon;
import javax.swing.text.{BadLocationException,Document}
import org.netbeans.api.lexer.{Token,TokenId,TokenHierarchy,TokenSequence}
import org.netbeans.editor.{BaseDocument,Utilities}
import org.netbeans.modules.csl.api.{ElementHandle,ElementKind,HtmlFormatter,Modifier,OffsetRange,StructureItem,StructureScanner}
import org.netbeans.modules.csl.api.StructureScanner._
import org.netbeans.modules.csl.spi.ParserResult
import org.netbeans.modules.erlang.editor.ast.{AstDfn,AstRootScope,AstScope}
import org.netbeans.modules.erlang.editor.lexer.{ErlangTokenId,LexUtil}
import org.openide.util.Exceptions

import scala.collection.mutable.ArrayBuffer

/**
 *
 * @author Caoyuan Deng
 */
class ErlangStructureAnalyzer extends StructureScanner {

    override
    def getConfiguration :Configuration = null

    override
    def scan(result:ParserResult) :List[StructureItem] = result match {
        case null => Collections.emptyList[StructureItem]
        case pResult:ErlangParserResult => 
            var items = Collections.emptyList[StructureItem]
            for (rootScope <- pResult.rootScope) {
                items = new ArrayList[StructureItem](rootScope.dfns.size)
                scanTopForms(rootScope, items, pResult)
            }
            items
    }

    private def scanTopForms(scope:AstScope, items:List[StructureItem], pResult:ErlangParserResult) :Unit = {
        for (dfn <- scope.dfns) {
            dfn.getKind match {
                case ElementKind.ATTRIBUTE | ElementKind.METHOD => items.add(new ErlangStructureItem(dfn, pResult))
                case _ =>
            }
            scanTopForms(dfn.bindingScope, items, pResult)
        }
    }

    override
    def folds(result:ParserResult) :Map[String, List[OffsetRange]] = result match {
        case null => Collections.emptyMap[String, List[OffsetRange]]
        case pResult:ErlangParserResult =>
            var folds = Collections.emptyMap[String, List[OffsetRange]]
            for (rootScope <- pResult.rootScope;
                 doc <- LexUtil.document(pResult, true);
                 th <- LexUtil.tokenHierarchy(pResult);
                 ts <- LexUtil.tokenSequence(th, 1)
            ) {
                folds = new HashMap[String, List[OffsetRange]]
                val codefolds = new ArrayList[OffsetRange]
                folds.put("codeblocks", codefolds); // NOI18N

                // * Read-lock due to Token hierarchy use
                doc.readLock

                addCodeFolds(pResult, doc, rootScope.dfns, codefolds)

                var lineCommentStart = 0
                var lineCommentEnd = 0
                var startLineCommentSet = false

                val comments = new Stack[Array[Integer]]
                val blocks = new Stack[Integer]

                while (ts.isValid && ts.moveNext) {
                    val token = ts.token
                    token.id match {
                        case ErlangTokenId.LineComment =>
                            val offset = ts.offset
                            if (!startLineCommentSet) {
                                lineCommentStart = offset
                                startLineCommentSet = true
                            }
                            lineCommentEnd = offset

                        case ErlangTokenId.Case | ErlangTokenId.If | ErlangTokenId.Try | ErlangTokenId.Receive =>
                            val blockStart = ts.offset
                            blocks.push(blockStart)

                            startLineCommentSet = false
                        
                        case ErlangTokenId.End if !blocks.empty =>
                            val blockStart = blocks.pop.asInstanceOf[Int]
                            val blockRange = new OffsetRange(blockStart, ts.offset + token.length)
                            codefolds.add(blockRange)

                            startLineCommentSet = false
                        case _ =>
                            startLineCommentSet = false
                    }
                }

                doc.readUnlock

                try {
                    /** @see GsfFoldManager#addTree() for suitable fold names. */
                    lineCommentEnd = Utilities.getRowEnd(doc, lineCommentEnd)

                    if (Utilities.getRowCount(doc, lineCommentStart, lineCommentEnd) > 1) {
                        val lineCommentsFolds = new ArrayList[OffsetRange];
                        val range = new OffsetRange(lineCommentStart, lineCommentEnd)
                        lineCommentsFolds.add(range)
                        folds.put("comments", lineCommentsFolds) // NOI18N
                    }
                } catch {
                    case ex:BadLocationException => Exceptions.printStackTrace(ex)
                }
            }

            folds
    }

    @throws(classOf[BadLocationException])
    private def addCodeFolds(pResult:ErlangParserResult, doc:BaseDocument, defs:ArrayBuffer[AstDfn], codeblocks:List[OffsetRange]) :Unit = {
        import ElementKind._
       
        for (dfn <- defs) {
            val kind = dfn.getKind
            kind match {
                case FIELD | METHOD | CONSTRUCTOR | CLASS | MODULE | ATTRIBUTE =>
                    var range = dfn.getOffsetRange(pResult)
                    var start = range.getStart
                    // * start the fold at the end of the line behind last non-whitespace, should add 1 to start after "->"
                    start = Utilities.getRowLastNonWhite(doc, start) + 1
                    val end = range.getEnd
                    if (start != -1 && end != -1 && start < end && end <= doc.getLength) {
                        range = new OffsetRange(start, end)
                        codeblocks.add(range)
                    }
                case _ =>
            }
    
            val children = dfn.bindingScope.dfns
            addCodeFolds(pResult, doc, children, codeblocks)
        }
    }

    private class ErlangStructureItem(val dfn:AstDfn, pResult:ParserResult) extends StructureItem {
        import ElementKind._

        override
        def getName :String = dfn.getName

        override
        def getSortText :String = getName

        override
        def getHtml(formatter:HtmlFormatter) :String = {
            dfn.htmlFormat(formatter)
            formatter.getText
        }

        override
        def getElementHandle :ElementHandle = dfn

        override
        def getKind :ElementKind = dfn.getKind
        
        override
        def getModifiers :Set[Modifier] = dfn.getModifiers

        override
        def isLeaf :Boolean = dfn.getKind match {
            case MODULE | CLASS => false
            case CONSTRUCTOR | METHOD | FIELD | VARIABLE | OTHER | PARAMETER | ATTRIBUTE => true
            case _ => true
        }

        override
        def getNestedItems : List[StructureItem] = {
            val nested = dfn.bindingScope.dfns
            if (nested.size > 0) {
                val children = new ArrayList[StructureItem](nested.size)

                for (child <- nested) {
                    child.kind match {
                        case PARAMETER | VARIABLE | OTHER =>
                        case _ => children.add(new ErlangStructureItem(child, pResult))
                    }
                }

                children
            } else Collections.emptyList[StructureItem]
        }

        override
        def getPosition :Long = {
            try {
                LexUtil.tokenHierarchy(pResult) match {
                    case None => 0
                    case Some(th) => dfn.boundsOffset(th)
                }
            } catch {case ex:Exception => 0}
        }

        override
        def getEndPosition :Long = {
            try {
                LexUtil.tokenHierarchy(pResult) match {
                    case None => 0
                    case Some(th) => dfn.boundsEndOffset(th)
                }
            } catch {case ex:Exception => 0}
        }

        override
        def equals(o:Any) :Boolean = o match {
            case null => false
            case x:ErlangStructureItem if dfn.getKind == x.dfn.getKind && getName.equals(x.getName) => true
            case _ => false
        }

        override
        def hashCode :Int = {
            var hash = 7
            hash = (29 * hash) + (if (getName != null) getName.hashCode else 0)
            hash = (29 * hash) + (if (dfn.getKind != null) dfn.getKind.hashCode else 0)
            hash
        }

        override
        def toString = getName

        override
        def getCustomIcon :ImageIcon = null
    }
}

Which just travels the rootScope, fetches AstDfn's instances and wrap them to ErlangStructureItem.

The last step is register this structure analyzer to ErlangLanguage.Scala as usual:

    override
    def hasStructureScanner = true

    override
    def getStructureScanner = new ErlangStructureAnalyzer

For structure analyzer, you need also to register it in layer.xml:

    <folder name="CslPlugins">
        <folder name="text">
            <folder name="x-erlang">
                <file name="language.instance">
                    <attr name="instanceClass" stringvalue="org.netbeans.modules.erlang.editor.ErlangLanguage"/>
                </file>
                <file name="structure.instance">
                    <attr name="instanceClass" stringvalue="org.netbeans.modules.erlang.editor.ErlangStructureAnalyzer"/>
                </file>
            </folder>
        </folder>
    </folder>

Build and run, now open a .erl file, you got:

Click on the picture to enlarge it

The functions/attributes are shown in the left-side navigator window now, there are also some code-folding marks. Did you notice my preliminaty error-recover work? :-)

With lexer integrated, you've got all tokens of source file. You can code for pretty formatting, indentation, auto pair inserting, pair matching based on tokens now. But I'll try to integrate an Erlang parser first to get the basic infrastructure ready.

I can choose to use Erlang's native compiler, than via jInterface, Scala can rpc call Erlang runtime to parse source file and return an AST tree. But, Erlang's library function for parsing is not good on error-recover currently, so I choose to write an Erlang parser in Rats!. Rats! generated parser is bad on error messages and error-recover too, but I know how to improve it.

Migrating a NetBeans Schliemann based grammar definition to Rats! is almost straightforward. For me, as I'm already familiar with Rats!, it's an one day work. The preliminary Rats! definition for Erlang can be found at ParserErlang.rats. Then, I rewrote a new ParserErlang.rats according to latest Erlang spec in another half day, without the LL(k) limitation, the grammar rules now keep as close as the original spec.

To get it be used to generate a ParserErlang.java, I added corresponding ant target to build.xml, which looks like:

    <target name="rats" depends="init" description="Scanner">
        <echo message="Rebuilding token scanner... ${rats.package.dir}"/>
        <java fork="yes"
             dir="${src.dir}/${rats.package.dir}"
             classname="xtc.parser.Rats"
             classpath="${rats.jar}">
            <arg value="-in"/>
            <arg value="${src.dir}"/>
            <arg value="${rats.lexer.file}"/>
        </java>

        <echo message="Rebuilding grammar parser... ${rats.package.dir}"/>
        <java fork="yes"
             dir="${src.dir}/${rats.package.dir}"
             classname="xtc.parser.Rats"
             classpath="${rats.jar}">
            <arg value="-in"/>
            <arg value="${src.dir}"/>
            <arg value="${rats.parser.file}"/>
        </java>
    </target>

And rats related properties, such as "rats.lexer.file" and "rats.parser.file" are defined in nbproject/project.properties as:

rats.jar=${cluster}/modules/xtc.jar
rats.package.dir=org/netbeans/modules/erlang/editor/rats
rats.lexer.file=LexerErlang.rats
rats.parser.file=ParserErlang.rats

Running target "rats" will generate LexerErlang.java and ParserErlang.java, the first one is that we've used in ErlangLexer.scala, the later one is to be integrated into NetBeans' Parsing API as the parser.

Now, you should extend two Parsing API abstract classes, org.netbeans.modules.csl.spi.ParserResult and org.netbeans.modules.parsing.spi.Parser. The first one will carry result's AST Node and syntax errors that parser detected, the errors will be highlighted automatically in Editor. The second one is the bridge between NetBeans parsing task and your real parser (ParserErlang.java here)

There are tricks to do some error-recover in these two classes, by adding a "." or "end" or "}" etc to the buffered source chars when a syntax error occurred, it's called "sanitize" the source to recover the error. NetBeans' Ruby, JavaScripts supporting have some good examples of this trick. For my case, I will do error recover in Rats! definition later, so I do not use this trick currently, but I leave some "sanitize" related code there.

First, it's an ErlangParserResult.scala which extended ParserResult. The code is simple at the first phase.

ErlangParserResult.scala

/*
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER.
 *
 * Copyright 1997-2007 Sun Microsystems, Inc. All rights reserved.
 *
 * The contents of this file are subject to the terms of either the GNU
 * General Public License Version 2 only ("GPL") or the Common
 * Development and Distribution License("CDDL") (collectively, the
 * "License"). You may not use this file except in compliance with the
 * License. You can obtain a copy of the License at
 * http://www.netbeans.org/cddl-gplv2.html
 * or nbbuild/licenses/CDDL-GPL-2-CP. See the License for the
 * specific language governing permissions and limitations under the
 * License.  When distributing the software, include this License Header
 * Notice in each file and include the License file at
 * nbbuild/licenses/CDDL-GPL-2-CP.  Sun designates this
 * particular file as subject to the "Classpath" exception as provided
 * by Sun in the GPL Version 2 section of the License file that
 * accompanied this code. If applicable, add the following below the
 * License Header, with the fields enclosed by brackets [] replaced by
 * your own identifying information:
 * "Portions Copyrighted [year] [name of copyright owner]"
 *
 * Contributor(s):
 *
 * The Original Software is NetBeans. The Initial Developer of the Original
 * Software is Sun Microsystems, Inc. Portions Copyright 1997-2006 Sun
 * Microsystems, Inc. All Rights Reserved.
 *
 * If you wish your version of this file to be governed by only the CDDL
 * or only the GPL Version 2, indicate your decision by adding
 * "[Contributor] elects to include this software in this distribution
 * under the [CDDL or GPL Version 2] license." If you do not indicate a
 * single choice of license, a recipient has the option to distribute
 * your version of this file under either the CDDL, the GPL Version 2 or
 * to extend the choice of license to its licensees as provided above.
 * However, if you add GPL Version 2 code and therefore, elected the GPL
 * Version 2 license, then the option applies only if the new code is
 * made subject to such option by the copyright holder.
 */
package org.netbeans.modules.erlang.editor

import _root_.java.util.{Collections, ArrayList, List}
import org.netbeans.api.lexer.{TokenHierarchy, TokenId}
import org.netbeans.modules.csl.api.Error
import org.netbeans.modules.csl.api.OffsetRange
import org.netbeans.modules.csl.spi.ParserResult
import org.netbeans.modules.parsing.api.Snapshot
import org.netbeans.modules.erlang.editor.rats.ParserErlang
import xtc.tree.{GNode}

/**
 *
 * @author Caoyuan Deng
 */
class ErlangParserResult(parser:ErlangParser,
                         snapshot:Snapshot,
                         val rootNode:GNode,
                         val th:TokenHierarchy[_]) extends ParserResult(snapshot) {

    override
    protected def invalidate :Unit = {
        // XXX: what exactly should we do here?
    }

    override
    def getDiagnostics :List[Error] = _errors

    private var _errors = Collections.emptyList[Error]
    
    def errors = _errors
    def errors_=(errors:List[Error]) = {
        this._errors = new ArrayList[Error](errors)
    }

    var source :String = _
    
    /**
     * Return whether the source code for the parse result was "cleaned"
     * or "sanitized" (modified to reduce chance of parser errors) or not.
     * This method returns OffsetRange.NONE if the source was not sanitized,
     * otherwise returns the actual sanitized range.
     */
    var sanitizedRange = OffsetRange.NONE
    var sanitizedContents :String = _
    var sanitized :Sanitize = NONE

    var isCommentsAdded :Boolean = false
    
    /**
     * Set the range of source that was sanitized, if any.
     */
    def setSanitized(sanitized:Sanitize, sanitizedRange:OffsetRange, sanitizedContents:String) :Unit = {
        this.sanitized = sanitized
        this.sanitizedRange = sanitizedRange
        this.sanitizedContents = sanitizedContents
    }

    override
    def toString = {
        "ErlangParseResult(file=" + snapshot.getSource.getFileObject + ",rootnode=" + rootNode + ")"
    }
}

Then, the ErlangParser.scala which extended Parser:

ErlangParser.scala

/*
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER.
 *
 * Copyright 1997-2007 Sun Microsystems, Inc. All rights reserved.
 *
 * The contents of this file are subject to the terms of either the GNU
 * General Public License Version 2 only ("GPL") or the Common
 * Development and Distribution License("CDDL") (collectively, the
 * "License"). You may not use this file except in compliance with the
 * License. You can obtain a copy of the License at
 * http://www.netbeans.org/cddl-gplv2.html
 * or nbbuild/licenses/CDDL-GPL-2-CP. See the License for the
 * specific language governing permissions and limitations under the
 * License.  When distributing the software, include this License Header
 * Notice in each file and include the License file at
 * nbbuild/licenses/CDDL-GPL-2-CP.  Sun designates this
 * particular file as subject to the "Classpath" exception as provided
 * by Sun in the GPL Version 2 section of the License file that
 * accompanied this code. If applicable, add the following below the
 * License Header, with the fields enclosed by brackets [] replaced by
 * your own identifying information:
 * "Portions Copyrighted [year] [name of copyright owner]"
 *
 * Contributor(s):
 *
 * The Original Software is NetBeans. The Initial Developer of the Original
 * Software is Sun Microsystems, Inc. Portions Copyright 1997-2006 Sun
 * Microsystems, Inc. All Rights Reserved.
 *
 * If you wish your version of this file to be governed by only the CDDL
 * or only the GPL Version 2, indicate your decision by adding
 * "[Contributor] elects to include this software in this distribution
 * under the [CDDL or GPL Version 2] license." If you do not indicate a
 * single choice of license, a recipient has the option to distribute
 * your version of this file under either the CDDL, the GPL Version 2 or
 * to extend the choice of license to its licensees as provided above.
 * However, if you add GPL Version 2 code and therefore, elected the GPL
 * Version 2 license, then the option applies only if the new code is
 * made subject to such option by the copyright holder.
 */
package org.netbeans.modules.erlang.editor


import _root_.java.io.{IOException, StringReader}
import _root_.java.util.ArrayList
import _root_.java.util.Collection
import _root_.java.util.List
import _root_.java.util.ListIterator
import _root_.javax.swing.event.ChangeListener
import _root_.javax.swing.text.BadLocationException

import org.netbeans.modules.csl.api.ElementHandle
import org.netbeans.modules.csl.api.Error
import org.netbeans.modules.csl.api.OffsetRange
import org.netbeans.modules.csl.api.Severity
import org.netbeans.modules.csl.spi.DefaultError
import org.netbeans.modules.parsing.api.Snapshot
import org.netbeans.modules.parsing.api.Task
import org.netbeans.modules.parsing.spi.ParseException
import org.netbeans.modules.csl.api.EditHistory
import org.netbeans.modules.csl.spi.GsfUtilities
import org.netbeans.modules.csl.spi.ParserResult
import org.netbeans.modules.parsing.api.Source
import org.netbeans.modules.parsing.spi.Parser
import org.netbeans.modules.parsing.spi.Parser.Result
import org.netbeans.modules.parsing.spi.ParserFactory
import org.netbeans.modules.parsing.spi.SourceModificationEvent
import org.openide.filesystems.FileObject
import org.openide.util.Exceptions

import org.netbeans.api.editor.EditorRegistry
import org.netbeans.api.lexer.{TokenHierarchy, TokenId}
import org.netbeans.editor.BaseDocument
import org.netbeans.modules.editor.NbEditorUtilities

import xtc.parser.{ParseError, SemanticValue}
import xtc.tree.{GNode, Location}

import org.netbeans.modules.erlang.editor.lexer.ErlangTokenId
import org.netbeans.modules.erlang.editor.rats.ParserErlang

/**
 *
 * @author Caoyuan Deng
 */
class ErlangParser extends Parser {

    private var lastResult :ErlangParserResult = _

    @throws(classOf[ParseException])
    override
    def parse(snapshot:Snapshot, task:Task, event:SourceModificationEvent) :Unit = {
        val context = new Context(snapshot, event)
        lastResult = parseBuffer(context, NONE)
        lastResult.errors = context.errors
    }

    @throws(classOf[ParseException])
    override
    def getResult(task:Task) :Result = {
        assert(lastResult != null, "getResult() called prior parse()") //NOI18N
        lastResult
    }

    override
    def cancel :Unit = {}

    override
    def addChangeListener(changeListener:ChangeListener) :Unit = {
        // no-op, we don't support state changes
    }

    override
    def removeChangeListener(changeListener:ChangeListener) :Unit = {
        // no-op, we don't support state changes
    }

    private def lexToAst(source:Snapshot, offset:Int) :Int = source match {
        case null => offset
        case _ => source.getEmbeddedOffset(offset)
    }

    private def astToLex(source:Snapshot, offset:Int) :Int = source match {
        case null => offset
        case _ => source.getOriginalOffset(offset)
    }

    private def sanitizeSource(context:Context, sanitizing:Sanitize) :Boolean = {
        false
    }

    private def sanitize(context:Context, sanitizing:Sanitize) :ErlangParserResult = {
        sanitizing match {
            case NEVER =>
                createParseResult(context)
            case NONE =>
                createParseResult(context)
            case _ =>
                // we are out of trick, just return as it
                createParseResult(context)
        }
    }

    protected def notifyError(context:Context, message:String, sourceName:String,
                              start:Int, lineSource:String, end:Int,
                              sanitizing:Sanitize, severity:Severity,
                              key:String, params:Object) :Unit = {

        val error = new DefaultError(key, message, null, context.fo, start, end, severity)

        params match {
            case null =>
            case x:Array[Object] => error.setParameters(x)
            case _ => error.setParameters(Array(params))
        }

        context.notifyError(error)

        if (sanitizing == NONE) {
            context.errorOffset = start
        }
    }

    protected def parseBuffer(context:Context, sanitizing:Sanitize) :ErlangParserResult = {
        var sanitizedSource = false
        var source = context.source

        sanitizing match {
            case NONE | NEVER =>
            case _ =>
                val ok = sanitizeSource(context, sanitizing)
                if (ok) {
                    assert(context.sanitizedSource != null)
                    sanitizedSource = true
                    source = context.sanitizedSource
                } else {
                    // Try next trick
                    return sanitize(context, sanitizing)
                }
        }

        if (sanitizing == NONE) {
            context.errorOffset = -1
        }

        val parser = createParser(context)

        val ignoreErrors = sanitizedSource
        var root :GNode = null
        try {
            var error :ParseError = null
            val r = parser.pS(0)
            if (r.hasValue) {
                val v = r.asInstanceOf[SemanticValue]
                root = v.value.asInstanceOf[GNode]
            } else {
                error = r.parseError
            }

            if (error != null && !ignoreErrors) {
                var start = 0
                if (error.index != -1) {
                    start = error.index
                }
                notifyError(context, error.msg, "Syntax error",
                            start, "", start,
                            sanitizing, Severity.ERROR,
                            "SYNTAX_ERROR", Array(error))

                System.err.println(error.msg)
            }

        } catch {
            case e:IOException => e.printStackTrace
            case e:IllegalArgumentException =>
                // An internal exception thrown by parser, just catch it and notify
                notifyError(context, e.getMessage, "",
                            0, "", 0,
                            sanitizing, Severity.ERROR,
                            "SYNTAX_ERROR", Array(e))
        }

        if (root != null) {
            context.sanitized = sanitizing
            context.root = root
            val r = createParseResult(context)
            r.setSanitized(context.sanitized, context.sanitizedRange, context.sanitizedContents)
            r.source = source
            r
        } else {
            sanitize(context, sanitizing)
        }
    }

    protected def createParser(context:Context) :ParserErlang = {
        val in = new StringReader(context.source)
        val fileName = if (context.fo != null) context.fo.getNameExt else ""

        val parser = new ParserErlang(in, fileName)
        context.parser = parser

        parser
    }

    private def createParseResult(context:Context) :ErlangParserResult = {
        new ErlangParserResult(this, context.snapshot, context.root, context.th)
    }

    /** Parsing context */
    class Context(val snapshot:Snapshot, event:SourceModificationEvent) {
        val errors :List[Error] = new ArrayList[Error]

        var source :String = ErlangParser.asString(snapshot.getText)
        var caretOffset :Int = GsfUtilities.getLastKnownCaretOffset(snapshot, event)

        var root :GNode = _
        var th :TokenHierarchy[_] = _
        var parser :ParserErlang = _
        var errorOffset :Int = _
        var sanitizedSource :String = _
        var sanitizedRange :OffsetRange = OffsetRange.NONE
        var sanitizedContents :String = _
        var sanitized :Sanitize = NONE

        def notifyError(error:Error) = errors.add(error)

        def fo = snapshot.getSource.getFileObject

        override
        def toString = "ErlangParser.Context(" + fo + ")" // NOI18N

    }
}

object ErlangParser {
    def asString(sequence:CharSequence) :String = sequence match {
        case s:String => s
        case _ => sequence.toString
    }

    def sourceUri(source:Source) :String = source.getFileObject match {
        case null => "fileless" //NOI18N
        case f => f.getNameExt
    }
}

/** Attempts to sanitize the input buffer */
sealed case class Sanitize
/** Only parse the current file accurately, don't try heuristics */
case object NEVER extends Sanitize
/** Perform no sanitization */
case object NONE extends Sanitize

The major tasks of these two classes are:

• Parsing the source buffer which is passed automatically by the framework when source is modified or just opened, the source text can be got from snapshot.getText. The parsing result used to be an AST tree, you can do semantic/structure analysis on AST tree to get more detailed information later, but as the first step, I just pass/keep AST tree in ErlangParserResult for later usage.
• Notifying the errors to framework. This can be done by store the parsing errors to ParserResult, and implemented getDiagnostics to return the error.

The last step is register the ErlangParser in ErlangLanguage.scala in one line code:

override def getParser = new ErlangParser

Now, open a .erl file, if there is syntax error, the editor will indicate and highlight it now.

My next step is to improve the Rats! definition, add error recover etc.

I'm going to create new Erlang Editor module in Scala, the module project is under http://hg.netbeans.org/main/contrib/file/0caa5d009839/erlang.editor/ which I just committed in.

The first step is to get Scala source file mixed in Java based NetBeans source/project tree. After define project's manifest,mf, we'll create a special build.xml for this project under erlang.editor directory:

build.xml

<?xml version="1.0" encoding="UTF-8"?>
<project name="contrib/erlang.editor" default="netbeans" basedir=".">
    <import file="../../nbbuild/templates/projectized.xml"/>
    <import file="scala-build.xml"/>

    <!-- special jar target for csl -->
    <target name="jar" depends="init,compile,jar-prep" unless="is.jar.uptodate">
        <taskdef name="csljar" classname="org.netbeans.modules.csl.CslJar" classpath="${nb_all}/csl.api/anttask/build/cslanttask.jar:${nb_all}/nbbuild/nbantext.jar"/>
        <csljar jarfile="${cluster}/${module.jar}" compress="${build.package.compress}" index="${build.package.index}" manifest="${manifest.mf}" stamp="${cluster}/.lastModified">
            <fileset dir="${build.classes.dir}"/>
        </csljar>
    </target>

    <target name="compile" depends="init,projectized-common.compile,scala-compile"/>
    <target name="do-test-build" depends="init,test-init,projectized-common.do-test-build"/>

    <target name="rats" depends="init" description="Scanner">
        <echo message="Rebuilding token scanner... ${rats.package.dir}"/>
        <java fork="yes"
             dir="${src.dir}/${rats.package.dir}"
             classname="xtc.parser.Rats"
             classpath="${rats.jar}">
            <arg value="-in"/>
            <arg value="${src.dir}"/>
            <arg value="${rats.lexer.file}"/>
        </java>
    </target>
    
</project>

This above build.xml imported NetBeans' main template ant xml "../../nbbuild/templates/projectized.xml" and a special scala-build.xml file, which defines scalac and javac task for mixed Scala/Java module for NetBeans. You can ignore the "rats" target if you has no plan to write language's lexer/parser in Rats! parser generator. scala-build.xml looks like:

scala-build.xml

<?xml version="1.0" encoding="UTF-8"?>
<project name="scala-module" default="netbeans" basedir=".">
    <import file="../../nbbuild/templates/projectized.xml"/>
    
    <target name="scala-taskdef" depends="init">
        <echo message="Compiling scala sources via ${scala.library}, ${scala.compiler}"/>
        <taskdef resource="scala/tools/ant/antlib.xml">
            <classpath>
                <pathelement location="${scala.library}"/>
                <pathelement location="${scala.compiler}"/>
            </classpath>
        </taskdef>
    </target>

    <property name="jar-excludes" value="**/*.java,**/*.form,**/package.html,**/doc-files/,**/*.scala"/>
    
    <target name="scala-compile" depends="init,up-to-date,scala-taskdef" unless="is.jar.uptodate">
        <!-- javac's classpath should include scala.library and all these paths of "cp" -->
        <path id="javac.cp">
            <pathelement path="${scala.libs}"/>
            <pathelement path="${module.classpath}"/>
            <pathelement path="${cp.extra}"/>
        </path>
        <!-- scalac will check class dependencies deeply, so we can not rely on public package only which is refed by ${module.classpath} -->
        <path id="scalac.cp">
            <pathelement path="${scala.libs}"/>
            <pathelement path="${module.run.classpath}"/>
            <pathelement path="${cp.extra}"/>
        </path>
        <mkdir dir="${build.classes.dir}"/>
        <depend srcdir="${src.dir}" destdir="${build.classes.dir}" cache="build/depcache">
            <classpath refid="scalac.cp"/>
        </depend>
        <!-- scalac -->
        <scalac srcdir="${src.dir}" destdir="${build.classes.dir}" encoding="UTF-8" target="jvm-${javac.target}">
            <classpath refid="scalac.cp"/>
        </scalac>
        <!-- javac -->
        <nb-javac srcdir="${src.dir}" destdir="${build.classes.dir}" debug="${build.compiler.debug}" debuglevel="${build.compiler.debuglevel}" encoding="UTF-8"
                deprecation="${build.compiler.deprecation}" optimize="${build.compiler.optimize}" source="${javac.source}" target="${javac.target}" includeantruntime="false">
            <classpath refid="javac.cp"/>
            <compilerarg line="${javac.compilerargs}"/>
            <processorpath refid="processor.cp"/>
        </nb-javac>
        <!-- Sanity check: -->
        <pathconvert pathsep=":" property="class.files.in.src">
            <path>
                <fileset dir="${src.dir}">
                    <include name="**/*.class"/>
                </fileset>
            </path>
        </pathconvert>
        <fail>
            <condition>
                <not>
                    <equals arg1="${class.files.in.src}" arg2=""/>
                </not>
            </condition>
            You have stray *.class files in ${src.dir} which you must remove.
            Probably you failed to clean your sources before updating them.
        </fail>
        <!-- OK, continue: -->
        <copy todir="${build.classes.dir}">
            <fileset dir="${src.dir}" excludes="${jar-excludes}"/>
        </copy>
    </target>

    <target name="do-test-build" depends="projectized-common.do-test-build">
        <scalac srcdir="${test.unit.src.dir}" destdir="${build.test.unit.classes.dir}" excludes="${test.excludes}"
               encoding="UTF-8">
            <classpath refid="test.unit.cp"/>
        </scalac>
    </target>
</project>

You need also to set the project's module dependencies on:

• org.netbeans.libs.scala
• org.netbeans.modules.csl.api
• org.netbeans.modules.lexer
• org.netbeans.modules.parsing.api
• org.openide.filesystems

And, the nbproject/project.properties which defines some important project building properties, such as:

javac.compilerargs=-Xlint:unchecked
javac.source=1.5
nbm.homepage=http://wiki.netbeans.org/Erlang

scala.library=${cluster}/modules/ext/scala-library-2.7.3.jar
scala.compiler=${cluster}/modules/ext/scala-compiler-2.7.3.jar
scala.libs=\
   ${scala.library}:\
   ${scala.compiler}

You can write NetBeans' module in Scala now.

>>> Updated Feb 8:

Code fixed to avoid an infinite cycled scanning when source file size > 16k

===

The minim supporting is to integrate a language lexer to NetBeans's language support framework. As NetBeans 7.0, there is a new effort for common language supporting, which is called CSL (Common Scripting Language). Don't be confused by this module's name, it not only for scripting language, actually, Java support has been migrated to CSL in 7.0. There are discussions on a better name. CSL is forked and created on GSF (Generic Scripting Framework) as a GSF's new variant that is based on new Parsing & Indexing API.

Since I'm going to write an Erlang lexer in Rats! generator parser, I need to add dependency on rats run-time libs first, rats run-time module is under contrib/xtc, which I patched to support end position of each production.

Then, you have to tell the project where to find the rats! libs, this can be done by adding following properties to nbproject/project.properties, now this nbproject/project.properties looks like:

nbproject/project.properties

javac.compilerargs=-Xlint:unchecked
javac.source=1.5
nbm.homepage=http://wiki.netbeans.org/Erlang

scala.library=${cluster}/modules/ext/scala-library-2.7.3.jar
scala.compiler=${cluster}/modules/ext/scala-compiler-2.7.3.jar
scala.libs=\
   ${scala.library}:\
   ${scala.compiler}

rats.jar=${cluster}/modules/xtc.jar
rats.package.dir=org/netbeans/modules/erlang/editor/rats
rats.lexer.file=LexerErlang.rats

All Rats! definitions of Erlang token can be found at http://hg.netbeans.org/main/contrib/file/tip/erlang.editor/src/org/netbeans/modules/erlang/editor/rats/. Don't ask me how to write Rats! rules for languages, you should get these information from Rats! web site. Or, you can integrate other types of lexer generated by other lexer generator, the examples can be found in NetBeans' other languages supporting modules.

The Erlang's lexer will be generated via "rats.lexer.file=LexerErlang.rats", which is the entry point of all defined rules for Erlang tokens. Run "rats" target will generate a LexerErlang.java file which is the lexer class that will be used to create Erlang tokens from Erlang source files.

Now, we should integrate this lexer class to NetBeans' lexer engine, this is done by two Scala files:

ErlangLexer.scala

/*
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER.
 *
 * Copyright 1997-2008 Sun Microsystems, Inc. All rights reserved.
 *
 * The contents of this file are subject to the terms of either the GNU
 * General Public License Version 2 only ("GPL") or the Common
 * Development and Distribution License("CDDL") (collectively, the
 * "License"). You may not use this file except in compliance with the
 * License. You can obtain a copy of the License at
 * http://www.netbeans.org/cddl-gplv2.html
 * or nbbuild/licenses/CDDL-GPL-2-CP. See the License for the
 * specific language governing permissions and limitations under the
 * License.  When distributing the software, include this License Header
 * Notice in each file and include the License file at
 * nbbuild/licenses/CDDL-GPL-2-CP.  Sun designates this
 * particular file as subject to the "Classpath" exception as provided
 * by Sun in the GPL Version 2 section of the License file that
 * accompanied this code. If applicable, add the following below the
 * License Header, with the fields enclosed by brackets [] replaced by
 * your own identifying information:
 * "Portions Copyrighted [year] [name of copyright owner]"
 *
 * Contributor(s):
 *
 * The Original Software is NetBeans. The Initial Developer of the Original
 * Software is Sun Microsystems, Inc. Portions Copyright 1997-2006 Sun
 * Microsystems, Inc. All Rights Reserved.
 *
 * If you wish your version of this file to be governed by only the CDDL
 * or only the GPL Version 2, indicate your decision by adding
 * "[Contributor] elects to include this software in this distribution
 * under the [CDDL or GPL Version 2] license." If you do not indicate a
 * single choice of license, a recipient has the option to distribute
 * your version of this file under either the CDDL, the GPL Version 2 or
 * to extend the choice of license to its licensees as provided above.
 * However, if you add GPL Version 2 code and therefore, elected the GPL
 * Version 2 license, then the option applies only if the new code is
 * made subject to such option by the copyright holder.
 */
package org.netbeans.modules.erlang.editor.lexer

import _root_.java.io.IOException
import _root_.java.io.Reader
import org.netbeans.api.lexer.{Token, TokenId}
import org.netbeans.modules.erlang.editor.rats.LexerErlang
import org.netbeans.spi.lexer.Lexer
import org.netbeans.spi.lexer.LexerInput
import org.netbeans.spi.lexer.LexerRestartInfo
import org.netbeans.spi.lexer.TokenFactory
import xtc.parser.Result
import xtc.tree.GNode
import xtc.util.Pair

import scala.collection.mutable.ArrayBuffer

import org.netbeans.modules.erlang.editor.lexer.ErlangTokenId._

/**
 *
 * @author Caoyuan Deng
 */
object ErlangLexer {
    /** @Note:
     * ErlangLexer class is not Reentrant safe, it seems when source size is large than 16 * 1024,
     * there will be more than one input are used, which causes the offset states, such as readed 
     * token length, offset etc in these inputs conflict?. Anyway it's safe to create a new one always.
     */
    def create(info:LexerRestartInfo[TokenId]) = new ErlangLexer(info)
}

class ErlangLexer(info:LexerRestartInfo[TokenId]) extends Lexer[TokenId] {
    /** @Note:
     * it seems input at this time is empty, so we can not do scanning here.
     * input will be filled in chars when call nextToken
     */

    var input :LexerInput = info.input
    var tokenFactory :TokenFactory[TokenId] = info.tokenFactory
    var lexerInputReader :LexerInputReader = new LexerInputReader(input)
    
    val tokenStream = new ArrayBuffer[TokenInfo]
    // * tokenStream.elements always return a new iterator, which point the first
    // * item, so we should have a global one.
    var tokenStreamItr :Iterator[TokenInfo]  = tokenStream.elements
    var lookahead :Int = 0

    override
    def release = {}

    override
    def state :Object = null

    override
    def nextToken :Token[TokenId] = {
        // * In case of embedded tokens, there may be tokens that had been scanned
        // * but not taken yet, check first
        if (!tokenStreamItr.hasNext) {
            tokenStream.clear
            scanTokens
            tokenStreamItr = tokenStream.elements

            /**
             * @Bug of LexerInput.backup(int) ?
             * backup(0) will cause input.readLength() increase 1
             */
            lookahead = input.readLength
            if (lookahead > 0) {
                // * backup all, we will re-read from begin to create token at following step
                input.backup(lookahead)
            } else {
                return null
            }
        }

        if (tokenStreamItr.hasNext) {
            val tokenInfo = tokenStreamItr.next

            if (tokenInfo.length == 0) {
                // * EOF
                return null
            }

            // * read token's chars according to tokenInfo.length
            var i = 0
            while (i < tokenInfo.length) {
                input.read
                i += 1
            }

            // * see if needs to lookahead, if true, perform it
            lookahead -= tokenInfo.length
            // * to cheat incremently lexer, we needs to lookahead one more char when
            // * tokenStream.size() > 1 (batched tokens that are not context free),
            // * so, when modification happens extractly behind latest token, will
            // * force lexer relexer from the 1st token of tokenStream
            val lookahead1 = if (tokenStream.size > 1) lookahead + 1 else lookahead
            if (lookahead1 > 0) {
                var i = 0
                while (i < lookahead1) {
                    input.read
                    i += 1
                }
                input.backup(lookahead1)
            }

            val tokenLength = input.readLength
            createToken(tokenInfo.id, tokenLength)
        } else {
            assert(false, "unrecognized input" + input.read)
            null
        }
    }

    def createToken(id:TokenId, length:Int) :Token[TokenId] = id.asInstanceOf[ErlangTokenId].fixedText match {
        case null => tokenFactory.createToken(id, length)
        case fixedText => tokenFactory.getFlyweightToken(id, fixedText)
    }

    def scanTokens :Result = {
        /**
         * We cannot keep an instance scope lexer, since lexer (sub-class of ParserBase)
         * has internal states which keep the read-in chars, index and others, it really
         * difficult to handle.
         */
        val scanner = new LexerErlang(lexerInputReader, "")
        try {
            // * just scan from position 0, incrmental lexer engine will handle start char in lexerInputReader
            val r = scanner.pToken(0)
            if (r.hasValue) {
                val node = r.semanticValue.asInstanceOf[GNode]
                flattenToTokenStream(node)
                r
            } else {
                System.err.println(r.parseError.msg)
                null
            }
        } catch {
            case e:Exception =>
                System.err.println(e.getMessage)
                null
        }
    }

    def flattenToTokenStream(node:GNode) :Unit = {
        val l = node.size
        if (l == 0) {
            /** @Note:
             * When node.size == 0, it's a void node. This should be limited to
             * EOF when you define lexical rats.
             *
             * And in Rats!, EOF is !_, the input.readLength() will return 0
             */      
            assert(input.readLength == 0,
                   "This generic node: " + node.getName +
                   " is a void node, this should happen only on EOF. Check you rats file.")

            val tokenInfo = new TokenInfo(0, null)
            tokenStream += tokenInfo
            return
        }
        
        var i = 0
        while (i < l) {
            node.get(i) match {
                case null =>
                    // * child may be null
                case child:GNode =>
                    flattenToTokenStream(child)
                case child:Pair[_] =>
                    assert(false, "Pair:" + child + " to be process, do you add 'flatten' option on grammar file?")
                case child:String =>
                    val length = child.length
                    val id = ErlangTokenId.valueOf(node.getName) match {
                        case None => ErlangTokenId.IGNORED
                        case Some(v) => v.asInstanceOf[TokenId]
                    }
          
                    val tokenInfo = new TokenInfo(length, id)
                    tokenStream += tokenInfo
                case child =>
                    println("To be process: " + child)
            }
            i += 1
        }
    }

    /**
     * Hacking for xtc.parser.ParserBase of Rats! which use java.io.Reader
     * as the chars input, but uses only {@link java.io.Reader#read()} of all methods in
     * {@link xtc.parser.ParserBase#character(int)}
     */
    class LexerInputReader(input:LexerInput) extends Reader {
        override
        def read :Int = input.read match {
            case LexerInput.EOF => -1
            case c => c
        }

        override
        def read(cbuf:Array[Char], off:Int, len:Int) :Int = {
            throw new UnsupportedOperationException("Not supported yet.")
        }

        override
        def close = {}
    }

    class TokenInfo(val length:Int, val id:TokenId) {
        override
        def toString = "(id=" + id + ", length=" + length + ")"
    }
}

ErlangTokenId

/*
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER.
 *
 * Copyright 1997-2007 Sun Microsystems, Inc. All rights reserved.
 *
 * The contents of this file are subject to the terms of either the GNU
 * General Public License Version 2 only ("GPL") or the Common
 * Development and Distribution License("CDDL") (collectively, the
 * "License"). You may not use this file except in compliance with the
 * License. You can obtain a copy of the License at
 * http://www.netbeans.org/cddl-gplv2.html
 * or nbbuild/licenses/CDDL-GPL-2-CP. See the License for the
 * specific language governing permissions and limitations under the
 * License.  When distributing the software, include this License Header
 * Notice in each file and include the License file at
 * nbbuild/licenses/CDDL-GPL-2-CP.  Sun designates this
 * particular file as subject to the "Classpath" exception as provided
 * by Sun in the GPL Version 2 section of the License file that
 * accompanied this code. If applicable, add the following below the
 * License Header, with the fields enclosed by brackets [] replaced by
 * your own identifying information:
 * "Portions Copyrighted [year] [name of copyright owner]"
 *
 * Contributor(s):
 *
 * The Original Software is NetBeans. The Initial Developer of the Original
 * Software is Sun Microsystems, Inc. Portions Copyright 1997-2006 Sun
 * Microsystems, Inc. All Rights Reserved.
 *
 * If you wish your version of this file to be governed by only the CDDL
 * or only the GPL Version 2, indicate your decision by adding
 * "[Contributor] elects to include this software in this distribution
 * under the [CDDL or GPL Version 2] license." If you do not indicate a
 * single choice of license, a recipient has the option to distribute
 * your version of this file under either the CDDL, the GPL Version 2 or
 * to extend the choice of license to its licensees as provided above.
 * However, if you add GPL Version 2 code and therefore, elected the GPL
 * Version 2 license, then the option applies only if the new code is
 * made subject to such option by the copyright holder.
 */
package org.netbeans.modules.erlang.editor.lexer

import _root_.java.util.Collection
import _root_.java.util.Collections
import _root_.java.util.HashMap
import _root_.java.util.HashSet
import _root_.java.util.Map
import _root_.java.util.Arrays

import org.netbeans.api.lexer.InputAttributes
import org.netbeans.api.lexer.Language
import org.netbeans.api.lexer.LanguagePath
import org.netbeans.api.lexer.Token
import org.netbeans.api.lexer.TokenId
import org.netbeans.spi.lexer.LanguageEmbedding
import org.netbeans.spi.lexer.LanguageHierarchy
import org.netbeans.spi.lexer.Lexer
import org.netbeans.spi.lexer.LexerRestartInfo

/**
 * 
 * @author Caoyuan Deng
 */
object ErlangTokenId extends Enumeration {
    // Let type of enum's value the same as enum itself
    type ErlangTokenId = V

    // Extends Enumeration.Val to get custom enumeration value
    class V(val name:String, val fixedText:String, val primaryCategory:String) extends Val(name) with TokenId {
        override
        def ordinal = id
    }
    object V {
        def apply(name:String, fixedText:String, primaryCategory:String) = new V(name, fixedText, primaryCategory)
    }
  
    val IGNORED = V("IGNORED", null, "ingore")
    val Error = V("Error", null, "error")

    // --- Spaces and comments
    val Ws = V("Ws", null, "whitespace")
    val Nl = V("Nl", null, "whitespace")
    val LineComment = V("LineComment", null, "comment")
    val CommentTag = V("CommentTag", null, "comment")
    val CommentData = V("CommentData", null, "comment")

    // --- Literals
    val IntegerLiteral = V("IntegerLiteral", null, "number")
    val FloatingPointLiteral = V("FloatingPointLiteral", null, "number")
    val CharacterLiteral = V("CharacterLiteral", null, "char")
    val StringLiteral = V("StringLiteral", null, "string")

    // --- Keywords
    val Andalso = V("Andalso", "andalso", "keyword")
    val After = V("After", "after", "keyword")
    val And = V("And", "and", "keyword")
    val Band = V("Band", "band", "keyword")
    val Begin = V("Begin", "begin", "keyword")
    val Bnot = V("Bnot", "bnot", "keyword")
    val Bor = V("Bor", "bor", "keyword")
    val Bsr = V("Bsr", "bsr", "keyword")
    val Bxor = V("Bxor", "bxor", "keyword")
    val Case = V("Case", "case", "keyword")
    val Catch = V("Catch", "catch", "keyword")
    val Cond = V("Cond", "cond", "keyword")
    val Div = V("Div", "div", "keyword")
    val End = V("End", "end", "keyword")
    val Fun = V("Fun", "fun", "keyword")
    val If = V("If", "if", "keyword")
    val Not = V("Not", "not", "keyword")
    val Of = V("Of", "of", "keyword")
    val Orelse = V("Orelse", "orelse", "keyword")
    val Or = V("Or", "or", "keyword")
    val Query = V("Query", "query", "keyword")
    val Receive = V("Receive", "receive", "keyword")
    val Rem = V("Rem", "rem", "keyword")
    val Try = V("Try", "try", "keyword")
    val Spec = V("Spec", "spec", "keyword")
    val When = V("When", "when", "keyword")
    val Xor = V("Xor", "xor", "keyword")

    // --- Identifiers
    val Macro = V("Macro", null, "identifier")
    val Atom = V("Atom", null, "identifier")
    val Var = V("Var", null, "identifier")
    val Rec = V("Rec", null, "identifier")

    // --- Stop
    val Stop = V("Stop", ".", "separator")

    // --- Symbols
    val LParen = V("LParen", "(", "separator")
    val RParen = V("RParan", ")", "separator")
    val LBrace = V("LBrace", "{", "separator")
    val RBrace = V("RBrace", "}", "separator")
    val LBracket = V("LBracket", "[", "separator")
    val RBracket = V("RBracket", "]", "separator")
    val Comma = V("Comma", ",", "separator")
    val Dot = V("Dot", ".", "separator")
    val Semicolon = V("Semicolon", ";", "separator")
    val DBar = V("DBar", "||", "separator")
    val Bar = V("Bar", "|",  "separator")
    val Question = V("Question", "?","separator")
    val DLt = V("DLt", "<<", "separator")
    val LArrow = V("LArrow", "<-", "separator")
    val Lt = V("Lt", "<", "separator")
    val DGt = V("DGt", ">>", "separator")
    val Ge = V("Ge", ">=", "separator")
    val Gt = V("Gt", ">", "separator")
    val ColonMinus = V("ColonMinus", ":-", "separator")
    val DColon = V("DColon", "::", "separator")
    val Colon = V("Colon", ":", "separator")
    val Hash = V("Hash", "#", "separator")
    val DPlus = V("DPlus", "++", "separator")
    val Plus = V("Plus", "+", "separator")
    val DMinus = V("DMinus", "--", "separator")
    val RArrow = V("RArrow", "->", "separator")
    val Minus = V("Minus", "-", "separator")
    val Star = V("Star", "*", "separator")
    val Ne = V("Ne", "/=", "separator")
    val Slash = V("Slash", "/", "separator")
    val EEq = V("EEq", "=:=", "separator")
    val ENe = V("ENe", "=/=", "separator")
    val DEq = V("DEq", "==", "separator")
    val Le = V("le", "=<", "separator")
    val Eq = V("Eq", "=", "separator")
    val Exclamation = V("Exclamation", "!", "separator")

  
    /**
     * MIME type for Erlang. Don't change this without also consulting the various XML files
     * that cannot reference this value directly.
     */
    val ERLANG_MIME_TYPE = "text/x-erlang"; // NOI18N

    // * should use "val" instead of "def" here to get a singleton language val, which  
    // * will be used to identity the token's language by "==" comparasion by other classes.
    // * Be aware of the init order! to get createTokenIds gathers all TokenIds, should
    // * be put after all token id val definition
    val language = new LanguageHierarchy[TokenId] {
        protected def mimeType = ERLANG_MIME_TYPE

        protected def createTokenIds :Collection[TokenId] = {
            val ids = new HashSet[TokenId]
            elements.foreach{ids add _.asInstanceOf[TokenId]}
            ids
        }
    
        protected def createLexer(info:LexerRestartInfo[TokenId]) :Lexer[TokenId] = ErlangLexer.create(info)

        override
        protected def createTokenCategories :Map[String, Collection[TokenId]] = {
            val cats = new HashMap[String, Collection[TokenId]]
            cats
        }

        override
        protected def embedding(token:Token[TokenId], languagePath:LanguagePath, inputAttributes:InputAttributes) = {
            null // No embedding
        }
    }.language

}

ErlangTokenId implemented org.netbeans.api.lexer.TokenId, and defined all Erlang token's Ids, with each name, id, fixedText and primaryCategory.

ErlangLexer.scala is the bridge between LexerErlang.java and NetBeans' lexer engine. NetBeans' lexer engine is an incremental engine, which will automatically handle the positions when you insert/modify a char, wrap the sanitary buffer in a LexerInput and pass it to lexer. I have carefully degined ErlangLexer.scala to get these benefits, you can use this file for other languages too, if you have understood how I write rats rules for tokens.

Now, you should implemented an org.netbeans.modules.csl.spi.DefaultLanguageConfig, which register all services for your language, such as: CodeCompletionHandler, DeclarationFinder, Formatter, IndexSearcher, InstantRenamer, KeystrokeHandler, OccurrencesFinder, SemanticAnalyzer, StructureScanner etc. For the first step, we only implemented a minim supporting for Erlang, which actually is only a lexer to get Erlang tokens and highlight them. So, our implementation is fairly simple:

ErlangLanguage.scala

/*
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER.
 *
 * Copyright 1997-2007 Sun Microsystems, Inc. All rights reserved.
 *
 * The contents of this file are subject to the terms of either the GNU
 * General Public License Version 2 only ("GPL") or the Common
 * Development and Distribution License("CDDL") (collectively, the
 * "License"). You may not use this file except in compliance with the
 * License. You can obtain a copy of the License at
 * http://www.netbeans.org/cddl-gplv2.html
 * or nbbuild/licenses/CDDL-GPL-2-CP. See the License for the
 * specific language governing permissions and limitations under the
 * License.  When distributing the software, include this License Header
 * Notice in each file and include the License file at
 * nbbuild/licenses/CDDL-GPL-2-CP.  Sun designates this
 * particular file as subject to the "Classpath" exception as provided
 * by Sun in the GPL Version 2 section of the License file that
 * accompanied this code. If applicable, add the following below the
 * License Header, with the fields enclosed by brackets [] replaced by
 * your own identifying information:
 * "Portions Copyrighted [year] [name of copyright owner]"
 *
 * Contributor(s):
 *
 * The Original Software is NetBeans. The Initial Developer of the Original
 * Software is Sun Microsystems, Inc. Portions Copyright 1997-2006 Sun
 * Microsystems, Inc. All Rights Reserved.
 *
 * If you wish your version of this file to be governed by only the CDDL
 * or only the GPL Version 2, indicate your decision by adding
 * "[Contributor] elects to include this software in this distribution
 * under the [CDDL or GPL Version 2] license." If you do not indicate a
 * single choice of license, a recipient has the option to distribute
 * your version of this file under either the CDDL, the GPL Version 2 or
 * to extend the choice of license to its licensees as provided above.
 * However, if you add GPL Version 2 code and therefore, elected the GPL
 * Version 2 license, then the option applies only if the new code is
 * made subject to such option by the copyright holder.
 */
package org.netbeans.modules.erlang.editor

import _root_.java.io.File
import _root_.java.util.Collection
import _root_.java.util.Collections
import _root_.java.util.HashMap
import _root_.java.util.Map
import _root_.java.util.Set
import org.netbeans.api.lexer.Language;
import org.netbeans.modules.csl.api.CodeCompletionHandler
import org.netbeans.modules.csl.api.DeclarationFinder
import org.netbeans.modules.csl.api.Formatter
import org.netbeans.modules.csl.api.IndexSearcher
import org.netbeans.modules.csl.api.InstantRenamer
import org.netbeans.modules.csl.api.KeystrokeHandler
import org.netbeans.modules.csl.api.OccurrencesFinder
import org.netbeans.modules.csl.api.SemanticAnalyzer
import org.netbeans.modules.csl.api.StructureScanner
import org.netbeans.modules.csl.spi.DefaultLanguageConfig
import org.netbeans.modules.parsing.spi.Parser
import org.netbeans.modules.parsing.spi.indexing.EmbeddingIndexerFactory
import org.openide.filesystems.FileObject
import org.openide.filesystems.FileUtil
import org.netbeans.modules.erlang.editor.lexer.ErlangTokenId

/*
 * Language/lexing configuration for Erlang
 *
 * @author Caoyuan Deng
 */
class ErlangLanguage extends DefaultLanguageConfig {

  override
  def getLexerLanguage = ErlangTokenId.language
    
  override
  def getDisplayName : String =  "Erlang"
    
  override
  def getPreferredExtension : String = {
    "erl" // NOI18N
  }    
}

The final step is to register ErlangLanguage and fontColor.xml, erlangResolver.xml etc in layer.xml for color highlights, mime resolver and language icon. All these necessary resource files are under: http://hg.netbeans.org/main/contrib/file/0caa5d009839/erlang.editor/src/org/netbeans/modules/erlang/editor/resources/

layer.xml

<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE filesystem PUBLIC "-//NetBeans//DTD Filesystem 1.1//EN" "http://www.netbeans.org/dtds/filesystem-1_1.dtd">
<filesystem>
    <folder name="Services">
        <folder name="MIMEResolver">
            <file name="Erlang.xml" url="erlangResolver.xml">
                <attr name="SystemFileSystem.localizingBundle" stringvalue="org.netbeans.modules.erlang.editor.resources.Bundle"/>
                <attr name="position" intvalue="1005"/>
            </file>
        </folder>
    </folder>

    <folder name="Editors">
      <!-- Reference binding color themes are under module: main/defaults/src/org/netbeans/modules/defaults -->
      <!-- color theme for nbeditor-settings-ColoringType -->
        <folder name="FontsColors">
            <folder name="Twilight">
                <folder name="Defaults">
                    <file name="org-netbeans-modules-defaults-highlight-colorings.xml" url="Twilight/editor.xml">
                        <attr name="SystemFileSystem.localizingBundle" stringvalue="org.netbeans.modules.defaults.Bundle"/>
                        <attr name="nbeditor-settings-ColoringType" stringvalue="highlight"/>
                    </file>
                    <file name="org-netbeans-modules-defaults-token-colorings.xml" url="Twilight/defaults.xml">
                        <attr name="SystemFileSystem.localizingBundle" stringvalue="org.netbeans.modules.defaults.Bundle"/>
                    </file>
                </folder>
            </folder>
            <folder name="EmacsStandard">
                <folder name="Defaults">
                    <file name="org-netbeans-modules-defaults-token-colorings.xml" url="EmacsStandard/defaults.xml">
                        <attr name="SystemFileSystem.localizingBundle" stringvalue="org.netbeans.modules.defaults.Bundle"/>
                    </file>
                </folder>
            </folder>
        </folder>

        <folder name="text">
            <folder name="x-erlang">
                <attr name="SystemFileSystem.localizingBundle" stringvalue="org.netbeans.modules.erlang.editor.Bundle"/>
                <file name="language.instance">
                    <attr name="instanceCreate" methodvalue="org.netbeans.modules.erlang.editor.lexer.ErlangTokenId.language"/>
                    <attr name="instanceOf" stringvalue="org.netbeans.api.lexer.Language"/>
                </file>
                
                <!-- TODO - this should not be necessary; I'm doing this now to work around
                    bugs in color initialization -->
                <folder name="FontsColors">
                    <folder name="NetBeans">
                        <folder name="Defaults">
                            <file name="coloring.xml" url="fontsColors.xml">
                                <attr name="SystemFileSystem.localizingBundle" stringvalue="org.netbeans.modules.erlang.editor.Bundle"/>
                            </file>
                        </folder>
                    </folder>
                    <folder name="Twilight">
                        <folder name="Defaults">
                            <file name="coloring.xml" url="Twilight/fontsColors.xml">
                                <attr name="SystemFileSystem.localizingBundle" stringvalue="org.netbeans.modules.erlang.editor.Bundle"/>
                            </file>
                        </folder>
                    </folder>
                    <folder name="EmacsStandard">
                        <folder name="Defaults">
                            <file name="coloring.xml" url="EmacsStandard/fontsColors.xml">
                                <attr name="SystemFileSystem.localizingBundle" stringvalue="org.netbeans.modules.erlang.editor.Bundle"/>
                            </file>
                        </folder>
                    </folder>
                </folder>

                <folder name="CodeTemplates">
                    <folder name="Defaults">
                        <file name="codeTemplates.xml" url="codeTemplates.xml">
                            <attr name="SystemFileSystem.localizingBundle" stringvalue="org.netbeans.modules.erlang.editor.Bundle"/>
                        </file>
                    </folder>
                </folder>
                <folder name="Keybindings">
                    <folder name="NetBeans">
                        <folder name="Defaults">
                            <file name="org-netbeans-modules-erlang-editor-keybindings.xml" url="keyBindings.xml"/>
                        </folder>
                    </folder>
                </folder>
            </folder>
        </folder>
    </folder>

    <folder name="CslPlugins">
        <folder name="text">
            <folder name="x-erlang">
                <file name="language.instance">
                    <attr name="instanceClass" stringvalue="org.netbeans.modules.erlang.editor.ErlangLanguage"/>
                </file>
            <!--file name="structure.instance">
               <attr name="instanceClass" stringvalue="org.netbeans.modules.scala.editing.ScalaStructureAnalyzer"/>
            </file-->
            </folder>
        </folder>
    </folder>
    
    <folder name="Loaders">
        <folder name="text">
            <folder name="x-erlang">
                <attr name="SystemFileSystem.icon" urlvalue="nbresloc:/org/netbeans/modules/erlang/editor/resources/Erlang.png"/>
                <attr name="iconBase" stringvalue="org/netbeans/modules/erlang/editor/resources/Erlang.png"/>
                <folder name="Actions">
                    <file name="OpenAction.instance">
                        <attr name="instanceClass" stringvalue="org.openide.actions.OpenAction"/>
                        <attr name="position" intvalue="100"/>
                    </file>
                    <file name="Separator1.instance">
                        <attr name="instanceClass" stringvalue="javax.swing.JSeparator"/>
                        <attr name="position" intvalue="200"/>
                    </file>
                    <file name="CutAction.instance">
                        <attr name="instanceClass" stringvalue="org.openide.actions.CutAction"/>
                        <attr name="position" intvalue="300"/>
                    </file>
                    <file name="CopyAction.instance">
                        <attr name="instanceClass" stringvalue="org.openide.actions.CopyAction"/>
                        <attr name="position" intvalue="400"/>
                    </file>
                    <file name="PasteAction.instance">
                        <attr name="instanceClass" stringvalue="org.openide.actions.PasteAction"/>
                        <attr name="position" intvalue="500"/>
                    </file>
                    <file name="Separator2.instance">
                        <attr name="instanceClass" stringvalue="javax.swing.JSeparator"/>
                        <attr name="position" intvalue="600"/>
                    </file>
                    <file name="NewAction.instance">
                        <attr name="instanceClass" stringvalue="org.openide.actions.NewAction"/>
                        <attr name="position" intvalue="700"/>
                    </file>
                    <file name="DeleteAction.instance">
                        <attr name="instanceClass" stringvalue="org.openide.actions.DeleteAction"/>
                        <attr name="position" intvalue="800"/>
                    </file>
                    <file name="RenameAction.instance">
                        <attr name="instanceClass" stringvalue="org.openide.actions.RenameAction"/>
                        <attr name="position" intvalue="900"/>
                    </file>
                    <file name="Separator3.instance">
                        <attr name="instanceClass" stringvalue="javax.swing.JSeparator"/>
                        <attr name="position" intvalue="1000"/>
                    </file>
                    <file name="SaveAsTemplateAction.instance">
                        <attr name="instanceClass" stringvalue="org.openide.actions.SaveAsTemplateAction"/>
                        <attr name="position" intvalue="1100"/>
                    </file>
                    <file name="Separator4.instance">
                        <attr name="instanceClass" stringvalue="javax.swing.JSeparator"/>
                        <attr name="position" intvalue="1200"/>
                    </file>
                    <file name="FileSystemAction.instance">
                        <attr name="instanceClass" stringvalue="org.openide.actions.FileSystemAction"/>
                        <attr name="position" intvalue="1300"/>
                    </file>
                    <file name="Separator5.instance">
                        <attr name="instanceClass" stringvalue="javax.swing.JSeparator"/>
                        <attr name="position" intvalue="1400"/>
                    </file>
                    <file name="ToolsAction.instance">
                        <attr name="instanceClass" stringvalue="org.openide.actions.ToolsAction"/>
                        <attr name="position" intvalue="1500"/>
                    </file>
                    <file name="PropertiesAction.instance">
                        <attr name="instanceClass" stringvalue="org.openide.actions.PropertiesAction"/>
                        <attr name="position" intvalue="1600"/>
                    </file>
                </folder>
            </folder>
        </folder>
    </folder>
</filesystem>

Now build your new module, lunch it, and open a .erl file, you get:

Click on the picture to enlarge it

ErlyBird (previous plugin) was written in Java, and based on NetBeans' Generic Languages Framework ( Project Schliemann). As NetBeans 7.0 introducing new  Parsing API, I'm going to migrate Erlang plugin for NetBeans to it, and rewrite in Scala.

Defining a Scala/Java mixed module project in NetBeans' trunk tree is a bit trick, but, since it's plain ant-based project, you can always take over it by writing/modifying build.xml. Challenges are actually from integrating Scala's Class/Object/Trait to an already existed framework. I met this kind of challenge quickly when I tried to implement an enum like class which should implement a Java interface org.netbeans.api.lexer.TokenId?

The signature of TokenId? is:

public interface TokenId {
    String name();
    int ordinal();
    String primaryCategory();
}

From the signature, you can get the hint to implement it as a Java enum. That's true and straightforward in Java. But how about in Scala?

There is an abstract class  Enumeration in Scala's library. The example code shows how to use it by extending it as a singleton Object. But for my case, I have to implement name() and ordinal() methods of TokenId? in the meantime, which brings some inconveniences for get the name() method automatically/magically for each enumeration value.

A quick google search gave some discussions about it, for example, a method reflection and invocation may be a choice to get name() simply implemented. I tried it, but finally dropped, because you cannot guarantee the condition of call stack of name(), it may happen to be called in this Object's own method, which then, may bring infinite cycled calls.

My final code is like:

package org.netbeans.modules.erlang.editor.lexer

object ErlangTokenId extends Enumeration {
  type ErlangTokenId = V

  // Extends Enumeration's inner class Val to get custom enumeration value
  class V(val name:String, val fixedText:String, val primaryCategory:String) extends Val(name) with TokenId {
    override
    def ordinal = id
  }
  object V {
    def apply(name:String, fixedText:String, primaryCategory:String) = new V(name, fixedText, primaryCategory)
  }
  
  val IGNORED = V("IGNORED", null, "ingore")
  val Error = V("Error", null, "error")

  // --- Spaces and comments
  val Ws = V("Ws", null, "whitespace")
  val Nl = V("Nl", null, "whitespace")
  val LineComment = V("LineComment", null, "comment")
  val CommentTag = V("CommentTag", null, "comment")
  val CommentData = V("CommentData", null, "comment")

  // --- Literals
  val IntegerLiteral = V("IntegerLiteral", null, "number")
  val FloatingPointLiteral = V("FloatingPointLiteral", null, "number")
  val CharacterLiteral = V("CharacterLiteral", null, "char")
  val StringLiteral = V("StringLiteral", null, "string")

  // --- Keywords
  val Andalso = V("Andalso", "andalso", "keyword")
  val After = V("After", "after", "keyword")
  val And = V("And", "and", "keyword")
  val Band = V("Band", "band", "keyword")
  val Begin = V("Begin", "begin", "keyword")
  val Bnot = V("Bnot", "bnot", "keyword")
  val Bor = V("Bor", "bor", "keyword")
  val Bsr = V("Bsr", "bsr", "keyword")
  val Bxor = V("Bxor", "bxor", "keyword")
  val Case = V("Case", "case", "keyword")
  val Catch = V("Catch", "catch", "keyword")
  val Cond = V("Cond", "cond", "keyword")
  val Div = V("Div", "div", "keyword")
  val End = V("End", "end", "keyword")
  val Fun = V("Fun", "fun", "keyword")
  val If = V("If", "if", "keyword")
  val Not = V("Not", "not", "keyword")
  val Of = V("Of", "of", "keyword")
  val Orelse = V("Orelse", "orelse", "keyword")
  val Or = V("Or", "or", "keyword")
  val Query = V("Query", "query", "keyword")
  val Receive = V("Receive", "receive", "keyword")
  val Rem = V("Rem", "rem", "keyword")
  val Try = V("Try", "try", "keyword")
  val When = V("When", "when", "keyword")
  val Xor = V("Xor", "xor", "keyword")

  // --- Identifiers
  val Atom = V("Atom", null, "identifier")
  val Var = V("Var", null, "identifier")

  // --- Symbols
  val LParen = V("LParen", "(", "separator")
  val RParen = V("RParan", ")", "separator")
  val LBrace = V("LBrace", "{", "separator")
  val RBrace = V("RBrace", "}", "separator")
  val LBracket = V("LBracket", "[", "separator")
  val RBracket = V("RBracket", "]", "separator")
  val Comma = V("Comma", ",", "separator")
  val Dot = V("Dot", ".", "separator")
  val Semicolon = V("Semicolon", ";", "separator")
  val DBar = V("DBar", "||", "separator")
  val Bar = V("Bar", "|",  "separator")
  val Question = V("Question", "?","separator")
  val DLt = V("DLt", "<<", "separator")
  val LArrow = V("LArrow", "<-", "separator")
  val Lt = V("Lt", "<", "separator")
  val DGt = V("DGt",  >", "separator")
  val Ge = V("Ge",  =", "separator")
  val Gt = V("Gt",  ", "separator")
  val ColonMinus = V("ColonMinus", ":-", "separator")
  val DColon = V("DColon", "::", "separator")
  val Colon = V("Colon", ":", "separator")
  val Hash = V("Hash", "#", "separator")
  val DPlus = V("DPlus", "++", "separator")
  val Plus = V("Plus", "+", "separator")
  val DMinus = V("DMinus", "--", "separator")
  val RArrow = V("RArrow", "->", "separator")
  val Minus = V("Minus", "-", "separator")
  val Star = V("Star", "*", "separator")
  val Ne = V("Ne", "/=", "separator")
  val Slash = V("Slash", "/", "separator")
  val EEq = V("EEq", "=:=", "separator")
  val ENe = V("ENe", "=/=", "separator")
  val DEq = V("DEq", "==", "separator")
  val Le = V("le", "=<", "separator")
  val Eq = V("Eq", "=", "separator")
  val Exclamation = V("Exclamation", "!", "separator")
}

First, I defined a class V which extends Enumeration.Val, and implemented TokenId with an extra field: fixedText.

Then, I have to explicitly put the value's name to this class and pass it to Enumeration.Val's constructor, so function ErlangTokenId.valueOf(String) will work as Java's enum type.

By type ErlangTokenId = V, type ErlangTokenId.V is now aliased as ErlangTokenId, so you can use ErlangTokenId instead of ErlangTokenId.V everywhere now, which exactly gets the effect of one of the behaviors of Java's enum: enum's value is the same type of enum itself.

• Bundling with Scala 2.7.3
• Partly supported Java/Scala mixed project: building project is OK (need to create a new Scala project); Java source is visible in Scala editor.
• Various bugs fixes

To download, please go to: https://sourceforge.net/project/showfiles.php?group_id=192439&package_id=256544&release_id=654650

For more information, please see http://wiki.netbeans.org/Scala

Bug reports are welcome.

=== Important Notice ===:

If you have a previous Scala plugin installed, after new plugins installed, you need to locate the NetBeans user profile:

• On Windows system, the default location is: "C:\Documents and Settings\<username>\.netbeans\<version-number>\"
• On Unix/Linux/MaxOS system: "<username>/.netbeans/<version-number>/"

=====================

In a previous blog: A Case Study of Scalability Related "Out of memory" Crash in Erlang, I described a scalability related issue, which was coming from a singleton async logger process. A singleton Erlang process can not benefit from multiple-core scalability.

I then did some testing on disk_log, which fortunately has sync log functions: log/2 and blog/2. Whenever a process calls disk_log:blog/2, the requesting process will monitor the logger process until it returns a result or failure. The piece of code is like:

monitor_request(Pid, Req) ->
    Ref = erlang:monitor(process, Pid),
    Pid ! {self(), Req},
    receive 
        {'DOWN', Ref, process, Pid, _Info} ->
            {error, no_such_log};
        {disk_log, Pid, Reply} ->
            erlang:demonitor(Ref),
            receive 
                {'DOWN', Ref, process, Pid, _Reason} ->
                    Reply
            after 0 ->
                    Reply
            end
    end.

Where Pid is the logger's process id.

This piece of code shows how to interactive synchronously between processes.

Under sync mode, there may be a lot of simultaneous requesting processes request the logger process to log message asynchronously, but each requesting process will wait the logger's work done before it requests next log, i.e. each requesting process requests the logger to log message synchronously. Upon this sync mode, we can guarantee the logger's message queue length won't exceed the number of simultaneous requesting processes.

I wrote some testing code: dlogger.erl

-module(dlogger).

-export([sync/2,
         async/2,
         proc_loop/4
        ]).

-define(LogName, blog).
-define(LogFile, "b.log").

% 100000, 10
sync(N_Msg, N_Procs) ->
   test(N_Msg, N_Procs, fun disk_log:blog/2).

async(N_Msg, N_Procs) ->
   test(N_Msg, N_Procs, fun disk_log:balog/2).


test(N_Msg, N_Procs, FunLog) ->
   MsgPerProc = round(N_Msg / N_Procs),
   Collector = init(N_Procs),
   LogPid = logger_pid(?LogName),
   io:format("logger pid: ~p~n", [LogPid]),
   Workers = [spawn(?MODULE, proc_loop, [Collector, MsgPerProc, LogPid, FunLog]) || _I <- lists:seq(1, N_Procs)],
   Start = now(),
   [Worker ! start || Worker <- Workers],
   %% don't terminate, wait here, until all tasks done.
   receive
      {sent_done, MaxMQLen, MaxMem} ->
         probe_logger(LogPid, MaxMQLen, MaxMem),
         [exit(Worker, kill) || Worker <- Workers],
         io:format("Time: ~10.2f ms~n", [timer:now_diff(now(), Start) / 1000])
   end.

init(N_Procs) ->
   disk_log:close(?LogName),
   disk_log:open([{name, ?LogName},
                  {file, ?LogFile},
                  {format, external}]),
   MainPid = self(),
   Collector = spawn(fun() -> collect(MainPid, N_Procs, 0, 0, 0, 0) end),
   Collector.

collect(MainPid, N_Procs, N_Finished, _N_Msg, MaxMQLen, MaxMem) when N_Procs == N_Finished ->
   MainPid ! {sent_done, MaxMQLen, MaxMem};
collect(MainPid, N_Procs, N_Finished, N_Msg, MaxMQLen, MaxMem) ->
   receive
      {Pid, sent_done, MQLen, _Mem} ->
         io:format("==== QLen ~p. Proc ~p finished, total finished: ~p ====~n", [MQLen, Pid, N_Finished + 1]),
         collect(MainPid, N_Procs, N_Finished + 1, N_Msg, MaxMQLen, MaxMem);
      {Pid, I, MQLen, Mem} ->
         %io:format("Processed/Qlen ~p/~p msgs. Logger mem is ~p. proc ~p: No.~p msgs sent~n", [N_Msg + 1, MQLen, Mem, Pid, I]),
         MaxMQLen1 = if MQLen > MaxMQLen -> MQLen; true -> MaxMQLen end,
         MaxMem1 = if Mem > MaxMem -> Mem; true -> MaxMem end,
         collect(MainPid, N_Procs, N_Finished, N_Msg + 1, MaxMQLen1, MaxMem1)
   end.

proc_loop(Collector, N_Msg, LogPid, LogFun) ->
   receive
      start ->
         do_proc_work(Collector, N_Msg, LogPid, LogFun, do_log)
   end.

do_proc_work(Collector, I, LogPid, LogFun, WorkType) ->
   Date = httpd_util:rfc1123_date(calendar:local_time()),
   MQLen = logger_mqlen(LogPid),
   Mem = logger_mem(LogPid),
   Msg = io_lib:format("logged in ~p, logger qlen is ~p, total mem is ~p\n",
                       [self(), MQLen, Mem]),
   Msg1 = list_to_binary([<<"=INFO REPORT==== ">>, Date, <<" ===\n">>, Msg, <<"\n">>]),

   WorkType1 = if WorkType == do_log ->
                     LogFun(?LogName, Msg1),
                     io:format("", []), % sync the io between collector if any 
                     Collector ! {self(), I, MQLen, Mem},
                     io:format("sent one msg, qlen:~p, mem:~p~n", [MQLen, Mem]),
                     if I =< 1 ->
                           Collector ! {self(), sent_done, MQLen, Mem},
                           io:format("~p sent done, qlen:~p, mem:~p~n", [self(), MQLen, Mem]),
                           keep_live;
                        true -> do_log
                     end;
                  true -> keep_live
               end,
   do_proc_work(Collector, I - 1, LogPid, LogFun, WorkType1).

probe_logger(Pid, MaxMQLen, MaxMem) ->
   MQLen = logger_mqlen(Pid),
   Mem = logger_mem(Pid),
   MaxMQLen1 = if MQLen > MaxMQLen -> MQLen; true -> MaxMQLen end,
   MaxMem1 = if Mem > MaxMem -> Mem; true -> MaxMem end,
   io:format("qlen is ~p, max qlen is ~p, max mem is ~p~n", [MQLen, MaxMQLen1, MaxMem1]),
   if MQLen == 0 -> done;
      true ->
         timer:sleep(10),
         probe_logger(Pid, MaxMQLen, MaxMem)
   end.

%% === helper ===
logger_pid(Log) ->
   case disk_log_server:get_log_pids(Log) of
      undefined ->
         undefined;
      {local, Pid} ->
         Pid;
      {distributed, [Pid|_Pids]} ->
         Pid
   end.

logger_mqlen(undefined) -> 0;
logger_mqlen(Pid) ->
   case process_info(Pid, message_queue_len) of
      {message_queue_len, Val} when is_integer(Val) -> Val;
      _ -> 0
   end.

logger_mem(undefined) -> 0;
logger_mem(Pid) ->
   case process_info(Pid, memory) of
      {memory, Val} when is_integer(Val) -> Val;
      _ -> 0
   end.

You can always use process_info/2 or process_info/1 to probe the information of a process. In above code, we will probe the logger process's message queue length and memory via logger_mqlen/1 and logger_mem/1 when necessary.

I wrote the code as it, so I can create thousands of processes first, then each process will repeatedly request to log message MsgPerProc? times, and keep alive after all messages have been sent.

And, to evaluate the actual task time, when all requesting processes have finished sending log messages, a probe_logger/1 function will confirm all messages in logger queue have been processed.

Here's the result:

> dlogger:sync(1000000, 1000) % 1 million log messages under 1000 requesting processes:
qlen is 0, max qlen is 999, max mem is 690,400
Time:  861286.24 ms

> dlogger:async(1000000, 1000) % 1 million log messages under 1000 requesting processes:
qlen is 0, max qlen is 68487, max mem is 75,830,616
Time: 2156351.45 ms

The performance in async mode is getting much worse comparing to sync mode. Under async mode, not only the elapsed time grew a lot, but also the max queue length reached to 68487, and the memory of logger process reached about 72M. Actually, after all messages had been sent, only a few messages had been processed by logger process. Since I kept 1000 processes alive, the logger process only <b>shared very very poor proportion CPU cycles</b>, the message processing procedure (log to disk buffer here) became very very slow, which caused the worse elapsed time. This case can be applied on any singleton process.

On the other side, under sync mode, the max queue length did not exceed the number of simultaneous requesting processes, here is 999 in case of 1000 simultaneous processes, and the max memory of logger process is reasonable in about 674K.

I'd like to emphasize the points:

• Singleton process in Erlang can not benefit from multiple-core scalability.
• Move code/job out of singleton process as much as possible to simultaneous processes, for example, before send the message to singleton process, done all pre-formatting/computing job.
• Some times, you need to sync the interacting between processes, to limit the singleton process working load.

We are building a platform for message switching, in Erlang. Everything looks OK on stability and features. It actually has run more than half year with zero down. We tested its performance on our 2-core CPU machine before, and got about 140 transactions/second, it's good enough.

Then, we got a 8-core CPU machine several weeks ago, and we did same performance testing on it, to see the scalability. Since Erlang is almost perfect on scalability, you can image the result, yes, about 700 transactions/second now, scaled almost linear. Until it crashed with "out of memory" when million hits processed.

It left a very big "erl_crash.dump" file there, I had to dig the issue. My first guess was, were some remote requests (access db, access remote web service etc) timeout but the process itself was not timeout yet, and cause more and more processes kept in VM?

A quick grep "=proc:" erl_crash.dump showed that the total number of processes was about 980, which was reasonable for our case.

So, which process ate so many memory? A quick grep "Stack+head" erl_crash.dump showed that there was indeed a process with 285082125 size of Stack+head there.

Following this clue, I caught this process:

=proc:<0.4.0>
State: Garbing
Name: error_logger
Spawned as: proc_lib:init_p/5
Last scheduled in for: io_lib_format:pad_char/2
Spawned by: <0.1.0>
Started: Sun Apr  1 01:21:50 2012
Message queue length: 2086029
Number of heap fragments: 1234053
Heap fragment data: 281266956
Link list: [<0.27.0>, <0.0.0>, {from,<0.42.0>,#Ref<0.0.0.88>}]
Reductions: 72745575
Stack+heap: 285082125
OldHeap: 47828850
Heap unused: 121777661
OldHeap unused: 47828850
Program counter: 0x0764c66c (io_lib_format:pad_char/2 + 4)
CP: 0x0764c1b4 (io_lib_format:collect_cseq/2 + 124)

This process was error_logger, which is from OTP/Erlang standard lib: error_logger, writing received messages to log file or tty. The typical usage is:

error_logger:info_msg("~p:~p " ++ Format, [?MODULE, ?LINE] ++ Data))

Which will format Data to a String according to the Format string, and write it to tty or log file.

he above case showed the message queue length of process "error_logger" had reached 1234053, and the Stack+heap was 285082125, about 272M size.

So the cause may be, that the message queue could not be processed in time, the messages were crowded in error_logger's process and finally caused "out of memory". The bottle-neck was that when error_logger tried to format the message to String, Erlang VM was weak on processing them, which seemed to need a lot of CPU cycles. In my previous blog, I talked about Erlang is bad on massive text processing. Erlang processes String/Text via List, which is obvious bottle-neck in Erlang now, with Erlang is getting much and much popular and more and more Erlang applications are written.

But, why this did not happen on our 2-core CPU machine? It's an interesting scalability related problem:

"error_logger" module will registered one and only one process to receive and handle all log messages. But Erlang VM's scheduler can not distribute ONE process to use multiple CPUs' computing ability. In our 2-core machine, the whole ability is about 140 transactions/second, the one process of "error_logger" just happened to have the power to handle corresponding log messages in time. Under 8-core CPUs machine, our platform scales to handle 700 transactions/second, but there is still only one process of "error_logger", which can not use 8-core CPUs' ability at all, and finally fail on it.

Erlang treats every process fairly (although you can change the priority manually), we can do a simple/quick evaluation:

1. 2-Core machine, keeping hits at 140 trans/second:

The number of simultaneous processes will be about 200, each process shares the CPU cycles: 1/200 * 2 Core = 1%

2. 8-Core machine, keeping hits at 700 trans/second:

The number of simultaneous processes will be about 980, each process shares the CPU cycles: 1/980 * 8 Core = 0.82%

So, the CPU cycles shared by error_logger process actually not increases. BTW, I think error_logger should cut its message queue when can not process them in time (disk IO may also be slower than receiving messages).

1==0.999999999......

Scala has extra bonus, to me, static types and OO/FP. The domains I face are usually with a lot of business logic, or, the worlds I try to describe are not only messages, they are, models I don't think are suitable to describe in Function only.

The world itself is OO/FP mixed, like Martin's quote: Two sides of coin. It's something like the Particle/Wave in Quantum. The world is an infinite whole, but the reason of Human Being is always finite, we are using out finite reason to measure the infinite world, it's an unsolvable contradiction: Infinity vs Finite. We have to read our world in OO and, in FP, in snapshot and in continuation.

There won't be "Super Hero" in computer languages, the world is getting self-organization and harmony, so do the languages. Each language is living in an eco-system, born, growing via interacting with environment, disappear ...

The Economy

My Self

I met Jackyz who is one of the translators of Chinese version "Programming Erlang". And Aimin who is writing a Delphi module to support Erlang c-node and c-driver in Delphi.

There is a commercial network monitoring product using Erlang from a major telecom company in China. And our Mobile-Banking platform (in Erlang) is scheduled to launch at middle of January too.

I talked with Yeka and Diuera from Broadview, a leading publisher in IT in China, they are really interested in importing "Programming in Scala" to mainland China.

And many thanks to Shiwei Xu, who is heavy working on Erlang community in China, and took the place to organize this conference.

I gave some encouragements to younger developers on learning Erlang and reading "Programming Erlang", since I'm the oldest one in attendees :-). Erlang is one of the best pragmatic and clear languages to learn concurrent/parallel and functional programming, and the book, is a very thoughtful and philosophic one on these perceptions.

And I'd like to see "Programming in Scala" also appeals in China soon, Scala is another pragmatic language on solving real world problems and, the book, is also thoughtful and philosophic one on our real world on Types, OO and FP.

Of course, choosing Scala or Erlang for your real world project should depend on the requirements.

I may be back to Vancouver next month for a while. Oh, it will be the beginning of new year.

CN Erlounge III photos by krzycube

Keeping Erlang in mind, I've coded two months in Scala, I'm thinking something called "Scala vs Erlang", I wrote some benchmark code to prove me (the code and result may be available someday), and I'd like to do some gradually summary on it in practical aspect. These opinions may be or not be correct currently due to lacking of deep experience and understanding, but, anyway, I need to record them now and correct myself with more experiences and understanding got on both Scala and Erlang.

Part I. Syntax

Keeping Erlang in mind, I've coded two months in Scala, I'm thinking something called "Scala vs Erlang", I wrote some benchmark code to prove me (the code and result may be available someday), and I'd like to do some gradually summary on it in practical aspect. These opinions may be or not be correct currently due to lacking of deep experience and understanding, but, anyway, I need to record them now and correct myself with more experiences and understanding got on both Scala and Erlang.

Part I. Syntax

List comprehension

Erlang:

Lst = [1,2,3,4],
[X + 1 || X <- Lst],
lists:map(fun(X) -> X + 1 end, Lst)

Scala:

val lst = List(1,2,3,4) 
for (x <- lst) yield x + 1
lst.map{x => x + 1}
lst.map{_ + 1} // or place holder

Pattern match

Erlang:

case X of
   {A, B} when is_integer(A), A > 1 -> ok;
   _ -> error
end,

{ok, [{A, B} = H|T]} = my_function(X)

Scala:

x match {
   case (a:Int, b:_) if a > 1 => OK // can match type
   case _ => ERROR
}

val ("ok", (h@(a, b)) :: t) = my_function(x)

List, Tuple, Array, Map, Binary, Bit

Erlang:

Lst = [1, 2, 3] %% List
[0 | Lst]  %% List concat
{1, 2, 3}  %% Tuple
<<1, 2, "abc">>  %% Binary
%% no Array, Map syntax

Scala:

val lst = List(1, 2, 3)  // List
0 :: lst  // List concat
(1, 2, 3) // Tuple
Array(1, 2, 3) // Array
Map("a" -> 1, "b" -> 2) // Map
// no Binary, Bit syntax

Process, Actor

Erlang:

the_actor(X) -> 
   receive 
      ok -> io:format("~p~n", [X]);
      I -> the_actor(X + I) %% needs to explicitly continue loop
   end.
P = spawn(mymodule, the_actor, [0])
P ! 1
P ! ok

Scala I:

class TheActor(x:Int) extends Actor { 
   def act = loop {
      react {
         case "ok" => println(x); exit // needs to explicitly exit loop
         case i:Int => x += i
      }
   }
}
val a = new TheActor(0)
a ! 1
a ! "ok"

Scala II:

val a = actor { 
   def loop(x:Int) = {
      react {
         case "ok" => println(x)
         case i:Int => loop(x + i)
      }
   }
   loop(0)
}
a ! 1
a ! "ok"

Part II. Processes vs Actors

Something I

Erlang:

• Lightweight processes
• You can always (almost) create a new process for each new comer
• Scheduler treats all processes fairly
• Share nothing between processes
• Lightweight context switch between processes
• IO has been carefully delegated to independent processes

Scala:

• Active actor is delegated to JVM thread, actor /= thread
• You can create a new actor for each new comer
• But the amount of real workers (threads) is dynamically adjusted according to the processing time
• The later comers may be in wait list for further processing until a spare thread is available
• Share nothing or share something upon you decision
• Heavy context switch between working threads
• IO block is still pain unless good NIO framework (Grizzly?)

Something II

Erlang:

• Try to service everyone simultaneously
• But may loss service quality when the work is heavy, may time out (out of service)
• Ideal when processing cost is comparable to context switching cost
• Ideal for small message processing in soft real-time
• Bad for massive data processing, and cpu-heavy work

Scala:

• Try to service limited number of customers best first
• If can not service all, the later comers will be put in waiting list and may time out (out of service)
• It's difficult for soft real-time on all coming concurrent customers
• Ideal when processing cost is far more than context switching cost (context switch time is in μs on modern JVM)
• When will there be perfect NIO + Actor library?

I'm pleased to announce Erlang plugin for NetBeans (ErlyBird) 0.17.0 is released.

This is a bug-fix release, and from now on, will be in form of NetBeans plugin.

NetBeans 6.5 is a requirement.

To download, please go to: https://sourceforge.net/project/showfiles.php?group_id=192439&package_id=226387&release_id=642911

To install:

1. Open NetBeans, go to "Tools" -> "Plugins", click on "Downloaded" tab title, click on "Add Plugins..." button, choose the directory where the Erlang plugin are unzipped, select all listed *.nbm files, following the instructions. Restart IDE.
2. Check/set your OTP path. From [Tools]->[Options], click on 'Miscellanous', then expand 'Erlang Installation', fill in the full path of your 'erl.exe' or 'erl' file. For instance: "C:/erl/bin/erl.exe"

When you open/create an Erlang project first time, the OTP libs will be indexed. The indexing time varies from 30 to 60 minutes depending on your computer.

Feedback and bug reports are welcome.

My Macbook is ân old one with Mac OS X 10.4, I have no way to get Java 6. I´ve tracked OpenSolaris? for a long time, with OpenSolaris? 08.11 is going to be released, I think I should have a try to see if I can do my daily work on OpenSolaris? rather than Mac OS on my MacBook?.

I then installed VirtualBox? on my MacOS, then a guest OpenSolaris? with 1024M memory and 16G disk. I downloaded and installed Java JDK 6 and NetBeans 6.5, plus my Scala plugins.

It rocks seamlessly. The only problem is, should I re-format my harddisk and install OpenSolaris? instead of Mac OS now?

There are a lot of guesses on Sun´s future these days, but, with all those innovations from Sun or taken by Sun, Why cloud-computing? Should still be sunshine-computing.

Click on the picture to enlarge it

• Much better code-completion
• Two new color themes: Twilight and Emacs Standard
• Various bugs fixes
• It's not perfect, but fairly stable
• Works good with NetBeans Maven plugin

To download, please go to: https://sourceforge.net/project/showfiles.php?group_id=192439&package_id=256544&release_id=641359

For more information, please see http://wiki.netbeans.org/Scala

Bug reports are welcome.

It works also on NetBeans RC2. If you have previous version of Scala plugin installed, you can upgrade to this version.

>>> Updated Nov 11:

Emacs Standard color theme already be there.

======

I'm beginning to write some real thing based on Liftweb. The more code I wrote, the more bugs of Scala plugin were fixed.

Not only bugs are being fixed, I also created a Twilight color theme for this plugin. And an Emacs color theme is also on the road.

When NetBeans 6.5 is official released, I'll put a new Scala plugin too.

Click on the picture to enlarge it

I packed a new Scala plugin, and tried several times to upload it to NetBeans' Plugins Portal and could not get job successfully done. So I uploaded it to sourceforge.net instead. The zip file is located at: [ https://sourceforge.net/project/showfiles.php?group_id=192439&package_id=256544&release_id=638797 scala-plugin-081106.zip

NetBeans' trunk has been targeting 7.0 now, the plugin on Last Development Build update center is no longer compatible with 6.5 RC2. That is, you should ignore my previous blog talked about installing plugin on 6.5 RC2, if you are using 6.5 RC2, you should download and install this one.

Nightly built version users can get latest plugin via NetBeans' plugin management feature as normal.

For more information, please visit  http://wiki.netbeans.org/Scala

This version fixed various bugs, and with some enhancements. The bundling Scala runtime is 2.7.2RC6, with NetBeans' maven plugin, it works on newest  Liftweb project.

Bug reports are always welcome.

Click on the picture to enlarge it

NetBeans 6.5 RC2 released. The Scala plugin on NetBeans' Plugins Portal is not compilable with RC2. To get Scala plugin working with RC2, do:

# Open NetBeans 6.5 RC2, go to "Tools" -> "Plugins", check "Setting" -> "Add", add new update center as "Last Development Build" with url: http://deadlock.netbeans.org/hudson/job/nbms-and-javadoc/lastStableBuild/artifact/nbbuild/nbms/updates.xml.gz

# Then in the "Available Plugins" tab, you can find the "Scala" category (or, you can click on "Name" in "Available Plugins" tab to find them. You may need to click "Reload Catalog" to get the latest available modules), check "Scala Kit" and click "Install", following the instructions. Restart IDE.

I'll re-pack a new version of Scala plugin for Plugins Portal when NetBeans 6.5 is officially released.

With more Scala coding experience, I refined color theme of Scala plugin for NetBeans. By desalinating Type name a bit, I found I can concentrate on logic a bit better. And all function calls are highlighted now, so, for multiple-line expression, when error-prone op is put on wrong new line, you can get some hints at once. It also gives you hint if a val/var is with implicit calling, which will be highlighted as a function call.

There are still some bugs when infer var/val's correct type in some cases.

Now the editor is much informative with highlighting and bubble pop display of type information (move mouse on it with CTRL/COMMAND pressed).

You'll need to update to newest 1.9.0 Editing module, please wait for it appealing on Development Update Center.

Click on the picture to enlarge it

There has been Java code in my previous blog:  RPC Server for Erlang, In Java, I'm now try to rewrite it in Scala. With the pattern match that I've been familiar with in Erlang, write the Scala version is really a pleasure. You can compare it with the  Java version.

I do not try Scala's actor lib yet, maybe late.

And also, I should port Erlang's jinterface to Scala, since OtpErlangTuple?, OtpErlangList? should be written in Scala's Tuple and List.

The code is auto-formatted by NetBeans' Scala plugin, and the syntax highlighting is the same as in NetBeans, oh, not exactly.

/*
 * RpcMsg.scala
 *
 */
package net.lightpole.rpcnode

import com.ericsson.otp.erlang.{OtpErlangAtom, OtpErlangList, OtpErlangObject, OtpErlangPid, OtpErlangRef, OtpErlangTuple}

class RpcMsg(val call:OtpErlangAtom,
             val mod :OtpErlangAtom,
             val fun :OtpErlangAtom,
             val args:OtpErlangList,
             val user:OtpErlangPid,
             val to  :OtpErlangPid,
             val tag :OtpErlangRef) {
}

object RpcMsg {
   
   def apply(msg:OtpErlangObject) : Option[RpcMsg] = msg match {
      case tMsg:OtpErlangTuple =>
         tMsg.elements() match {
            /* {'$gen_call', {To, Tag}, {call, Mod, Fun, Args, User}} */
            case Array(head:OtpErlangAtom, from:OtpErlangTuple, request:OtpErlangTuple) =>
               if (head.atomValue.equals("$gen_call")) {
                  (from.elements, request.elements) match {
                     case (Array(to :OtpErlangPid,
                                 tag:OtpErlangRef), Array(call:OtpErlangAtom,
                                                          mod :OtpErlangAtom,
                                                          fun :OtpErlangAtom,
                                                          args:OtpErlangList,
                                                          user:OtpErlangPid)) =>
                        if (call.atomValue.equals("call")) {
                           Some(new RpcMsg(call, mod, fun, args, user, to, tag))
                        } else None
                     case _ => None
                  }
               } else None
            case _ => None
         }
      case _ => None
   }
}