Thread: Detecting "letter-like" UTF-8 code points

So I'm currently working on this project which I've already added some very rudimentary UTF-8 support. What I'd really like to do though is be able to detect code points which represent conceptual "letter" types. So far I've only seen vague references to doing this sort of thing in the specs (https://www.unicode.org/versions/Uni...04.pdf#G134153).

Here's a toy program to show basically where I'm at on this:

Code:

#include <stdio.h>
#include <stdlib.h>
#include <stdbool.h>

typedef unsigned long utf8_rune;

void output(utf8_rune glyph)
{
 bool is_letter_like = false;
/*
 ...somehow detect valid letter types here...
*/
 printf("0x%zu: %s\n", glyph, is_letter_like ? "true" : "false");
}

bool output_utf8_glyphs(void* utf8, size_t length)
{
 typedef unsigned char byte;
 byte* next = utf8;
 byte* end = next + length; 
 for(;;)
 {
  if(next == end)
   break;
  byte octet = *next++;
  utf8_rune glyph = octet;  
  if(octet > 0x7f)
  {
   if(octet < 0xe0)
   {
    if((next + 1) > end)
     return false;
    glyph |= ((*next++) << 8); 
   }
   else if(octet < 0xf0)
   {
    if((next + 2) > end)
     return false;
    glyph |= ((*next++) << 8); 
    glyph |= ((*next++) << 16); 
   }
   else if(octet < 0xf8)
   {
    if((next + 3) > end)
     return false;
    glyph |= ((*next++) << 8); 
    glyph |= ((*next++) << 16);
    glyph |= ((*next++) << 24);     
   }
  } 
  output(glyph);   
 }
 return true;
}

int main(int argc, char** argv)
{
 if(argc == 1)
 {
  char utf8[] = 
"  Οὐχὶ ταὐτὰ παρίσταταί μοι γιγνώσκειν, ὦ ἄνδρες ᾿Αθηναῖοι,\n"
"  ὅταν τ᾿ εἰς τὰ πράγματα ἀποβλέψω καὶ ὅταν πρὸς τοὺς\n"
"  λόγους οὓς ἀκούω· τοὺς μὲν γὰρ λόγους περὶ τοῦ\n"
"  τιμωρήσασθαι Φίλιππον ὁρῶ γιγνομένους, τὰ δὲ πράγματ᾿\n" 
"  εἰς τοῦτο προήκοντα,  ὥσθ᾿ ὅπως μὴ πεισόμεθ᾿ αὐτοὶ\n"
"  πρότερον κακῶς σκέψασθαι δέον. οὐδέν οὖν ἄλλο μοι δοκοῦσιν\n"
"  οἱ τὰ τοιαῦτα λέγοντες ἢ τὴν ὑπόθεσιν, περὶ ἧς βουλεύεσθαι,\n"
"  οὐχὶ τὴν οὖσαν παριστάντες ὑμῖν ἁμαρτάνειν. ἐγὼ δέ, ὅτι μέν\n"
"  ποτ᾿ ἐξῆν τῇ πόλει καὶ τὰ αὑτῆς ἔχειν ἀσφαλῶς καὶ Φίλιππον\n"
"  τιμωρήσασθαι, καὶ μάλ᾿ ἀκριβῶς οἶδα· ἐπ᾿ ἐμοῦ γάρ, οὐ πάλαι\n"
"  γέγονεν ταῦτ᾿ ἀμφότερα· νῦν μέντοι πέπεισμαι τοῦθ᾿ ἱκανὸν\n"
"  προλαβεῖν ἡμῖν εἶναι τὴν πρώτην, ὅπως τοὺς συμμάχους\n"
"  σώσομεν. ἐὰν γὰρ τοῦτο βεβαίως ὑπάρξῃ, τότε καὶ περὶ τοῦ\n"
"  τίνα τιμωρήσεταί τις καὶ ὃν τρόπον ἐξέσται σκοπεῖν· πρὶν δὲ\n"
"  τὴν ἀρχὴν ὀρθῶς ὑποθέσθαι, μάταιον ἡγοῦμαι περὶ τῆς\n"
"  τελευτῆς ὁντινοῦν ποιεῖσθαι λόγον.\n";
  if(!output_utf8_glyphs(utf8, sizeof(utf8)))
   puts("Invalid unicode!"); 
 }
 else for(;;)
 {
  char* path = *(++argv);
  if(!path)
   break;  
  FILE* stream = fopen(path, "rb");
  if(!stream)
  {
   fprintf(stderr, "Error: cannot process file '%s'\n", path);
   continue;
  }
  fseek(stream, 0, SEEK_END);
  size_t size = ftell(stream);
  rewind(stream);
  char* utf8 = malloc(size + 1);
  fread(utf8, 1, size, stream);
  utf8[size] = 0;
  fclose(stream); 
  if(!output_utf8_glyphs(utf8, size))
   puts("Invalid unicode!");
  free(utf8);
 }
 return 0;
}