tokens_output.txt

1. FILEPATH

testdb=# SELECT ts_debug('/stuff/index.html');
                                     ts_debug                                   
  
--------------------------------------------------------------------------------
--
 (file,"File or path name",/stuff/index.html,{simple},simple,{/stuff/index.html}
)
 (blank,"Space symbols",/,{},,)
 (asciiword,"Word, all ASCII",stuff,{english_stem},english_stem,{stuff})
 (blank,"Space symbols",/,{},,)
 (asciiword,"Word, all ASCII",index,{english_stem},english_stem,{index})
 (blank,"Space symbols",.,{},,)
 (asciiword,"Word, all ASCII",html,{english_stem},english_stem,{html})


SELECT to_tsvector('english', '/stuff/index.html');
                    to_tsvector                     
----------------------------------------------------
 '/stuff/index.html':0 'html':2 'index':1 'stuff':0
(1 row)

2. URL

testdb=# SELECT ts_debug('http://example.com/stuff/index.html');
                                       ts_debug                                 
       
--------------------------------------------------------------------------------
-------
 (protocol,"Protocol head",http://,{},,)
 (url,URL,example.com/stuff/index.html,{simple},simple,{example.com/stuff/index.
html})
 (host,Host,example.com,{simple},simple,{example.com})
 (asciiword,"Word, all ASCII",example,{english_stem},english_stem,{exampl})
 (blank,"Space symbols",.,{},,)
 (asciiword,"Word, all ASCII",com,{english_stem},english_stem,{com})
 (url_path,"URL path",/stuff/index.html,{simple},simple,{/stuff/index.html})
 (blank,"Space symbols",/,{},,)
 (asciiword,"Word, all ASCII",stuff,{english_stem},english_stem,{stuff})
 (blank,"Space symbols",/,{},,)
 (asciiword,"Word, all ASCII",index,{english_stem},english_stem,{index})
 (blank,"Space symbols",.,{},,)
 (asciiword,"Word, all ASCII",html,{english_stem},english_stem,{html})
(13 rows)

testdb=# SELECT to_tsvector('english', 'http://example.com/stuff/index.html');
                                                      to_tsvector               
                                        
--------------------------------------------------------------------------------
----------------------------------------
 '/stuff/index.html':2 'com':1 'exampl':0 'example.com':0 'example.com/stuff/ind
ex.html':0 'html':4 'index':3 'stuff':2

3. EMAIL

testdb=# SELECT ts_debug('sushant@foo.bar');
                                  ts_debug                                   
-----------------------------------------------------------------------------
 (email,"Email address",sushant@foo.bar,{simple},simple,{sushant@foo.bar})
 (asciiword,"Word, all ASCII",sushant,{english_stem},english_stem,{sushant})
 (blank,"Space symbols",@,{},,)
 (asciiword,"Word, all ASCII",foo,{english_stem},english_stem,{foo})
 (blank,"Space symbols",.,{},,)
 (asciiword,"Word, all ASCII",bar,{english_stem},english_stem,{bar})


testdb=# SELECT to_tsvector('english', 'sushant@foo.bar');
                   to_tsvector                   
-------------------------------------------------
 'bar':2 'foo':1 'sushant':0 'sushant@foo.bar':0


4. HOST

testdb=# SELECT ts_debug('foo.bar.com');
                              ts_debug                               
---------------------------------------------------------------------
 (host,Host,foo.bar.com,{simple},simple,{foo.bar.com})
 (asciiword,"Word, all ASCII",foo,{english_stem},english_stem,{foo})
 (blank,"Space symbols",.,{},,)
 (asciiword,"Word, all ASCII",bar,{english_stem},english_stem,{bar})
 (blank,"Space symbols",.,{},,)
 (asciiword,"Word, all ASCII",com,{english_stem},english_stem,{com})

testdb=# SELECT to_tsvector('english', 'foo.bar.com');
               to_tsvector               
-----------------------------------------
 'bar':1 'com':2 'foo':0 'foo.bar.com':0