字符编码

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80

//
// Created by dingjing on 3/28/22.
//
#include <stdio.h>
#include <string.h>
#include <stdint.h>
#include <stdbool.h>

/**
 * 总共 94 个区 94 个位
 */
bool is_gb2312 (const char* code)
{
    int len = strlen (code);
    if (len != 2 && len != 1) {
        return false;
    }

    uint8_t codeH = code[0];
    uint8_t codeL = code[1];

    uint8_t area = codeH - 0xA0;
    uint8_t pos  = codeL - 0xA0;

    if ((len == 1) && (0 == (codeL & 0x80))) {
        printf ("ascii code: '%-3c' hex code: 0x%-2X\n", *code, *code);
        return true;
    }

    if (area >= 1 && area <= 94 && pos >= 1 && pos <= 94) {
        printf ("gb2312 code: '%-3s' -- area: %-2u, pos: %-2u hex code: 0x%-2X 0x%-2X\n", code, area, pos, codeH, codeL);
        return true;
    }
}


int main (int argc, char* argv[])
{
    /**
     * https://uic.io/en/charset/show_raw/gb2312
     * http://tools.jb51.net/table/gb2312
     * 1. '残'   十六进制: B2D0
     * 2. '怖'   十六进制: B2C0
     * 3. '惭'   十六进制: B2D1
     *
     * 4. ascii 里显示的特殊符号 32(space) - 47(/)
     * 5. ascii 里数字 30(0) - 39(9)
     * 6. ascii 里显示的特殊符号 58(:) - 64(@)
     * 7. ascii 里显示的大写字母 65(A) - 90(Z)
     * 8. ascii 里显示的标号字符 91([) - 96(`)
     * 9. ascii 里显示的大写字母 97(a) - 122(z)
     * 10. ascii 其它可显示字符 123({) - 126(~)
     *
     * uint16_t 就足够了
     */
    const char* arr[] = {
            "、", "∩", "〓",                              /* 第一区 */
            "残", "怖", "惭", "庄", "丽", "君", "齄",
            " ", "/",
            "0", "9",
            ":", "@",
            "A", "Z",
            "[", "`",
            "a", "z",
            "{", "~"
    };

    printf ("array: %lu/%lu\n\n", sizeof arr[0], sizeof arr);

    int len = sizeof arr / sizeof arr[0];
    for (int i = 0; i < len; ++i) {
        const char* code = arr[i];

        if (!is_gb2312 (code)) {
            printf ("index: %d -- code '%-3s' is not gb2312\n", i, code);
        }
    }

    return 0;
}

1

gcc -O0 -finput-charset=utf8 -fexec-charset=gb2312 demo-gb2312.run demo-gb2312.c

1
2
3
4
5
6

1字节 0xxxxxxx
2字节 110xxxxx 10xxxxxx
3字节 1110xxxx 10xxxxxx 10xxxxxx
4字节 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx
5字节 111110xx 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx
6字节 1111110x 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx

1
2
3
4
5
6
7
8
9

1字节, utf8-1 0x00-0x7F
2字节, utf8-2 0xC2-0XDF   0X80-0XBF
3字节, utf8-3 0xE0        0xA0-0XBF   0x80-0xBF
             0xE1-0XEC   0x80-0XBF   0x80-0xBF
             0xED        0x80-0x9F   0x80-0xBF
             0xEE-0xEF   0x80-0xBF   0x80-0xBF
4字节, utf8-4 0xF0        0x90-0xBF   0x80-0xBF   0x80-0xBF
             0xF1-0xF3   0x80-0xBF   0x80-0xBF   0x80-0xBF
             0xF4        0x80-0x8F   0x80-0xBF   0x80-0xBF

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148

//
// Created by dingjing on 3/28/22.
//

#include <stdio.h>
#include <stdint.h>
#include <string.h>
#include <stdbool.h>

bool is_utf8_1 (const char* code)
{
    int len = strlen(code);
    if (len < 1) {
        return false;
    }
    const uint8_t code1 = code[0];

    if (!(0x80 & code1)) {
        return true;
    }

    return false;
}

bool is_utf8_2 (const char* code)
{
    int len = strlen(code);
    if (len < 2) {
        return false;
    }
    const uint8_t code1 = code[0];
    const uint8_t code2 = code[1];

//    printf ("debug2: 0x%-4X\n", code);

    if (!is_utf8_1 (code) && (0xC0 == (0xE0 & code1)) && (0x80 == (0xE0 & code2))) {
//        printf ("debug: is utf8-2\n");
        if (code1 >= 0xC0 && code1 <= 0xDF && code2 >= 0x80 && code2 <= 0xBF) {
            return true;
        }
    }

    return false;
}

bool is_utf8_3 (const char* code)
{
    /**
     * 0xE0        0xA0-0XBF   0x80-0xBF
     * 0xE1-0XEC   0x80-0XBF   0x80-0xBF
     * 0xED        0x80-0x9F   0x80-0xBF
     * 0xEE-0xEF   0x80-0xBF   0x80-0xBF
     */
    int len = strlen(code);
    if (len < 3) {
        return false;
    }

    const uint8_t code1 = code[0];
    const uint8_t code2 = code[1];
    const uint8_t code3 = code[2];

    if ((0xE0 == code1) && ((code2 >= 0xA0) && (code2 <= 0XBF)) && ((code3 >= 0x80) && (code3 <= 0xBF))) {
        return true;
    } else if (((code1 >= 0xE1) && (code1 <= 0xEC)) && ((code2 >= 0x80) && (code2 <= 0xBF)) && ((code3 >= 0x80) && (code3 <= 0xBF))) {
        return true;
    } else if ((0xED == code1) && ((code2 >= 0x80) && (code2 <= 0x9F)) && ((code3 >= 0x80) && (code3 <= 0xBF))) {
        return true;
    } else if (((code1 >= 0xEE) && (code1 <= 0xEF)) && ((code2 >= 0x80) && (code2 <= 0xBF)) && ((code3 >= 0x80) && (code3 <= 0xBF))) {
        return true;
    }

    return false;
}

bool is_utf8_4 (const char* code)
{
    /**
     * 0xF0        0x90-0xBF   0x80-0xBF   0x80-0xBF
     * 0xF1-0xF3   0x80-0xBF   0x80-0xBF   0x80-0xBF
     * 0xF4        0x80-0x8F   0x80-0xBF   0x80-0xBF
     */
    int len = strlen(code);
    if (len < 4) {
        return false;
    }

    const uint8_t code1 = code[0];
    const uint8_t code2 = code[1];
    const uint8_t code3 = code[2];
    const uint8_t code4 = code[3];

//    printf ("debug4: 0x%-4X %-4X\n", code1H, code1L);

    if (!is_utf8_1 (code) && !is_utf8_2 (code) && !is_utf8_3 (code)) {
        if ((0xF0 == code1)
            && ((code2 >= 0x90) && (code2 <= 0xBF))
            && ((code3 >= 0x80) && (code3 <= 0xBF))
            && ((code4 >= 0x80) && (code4 <= 0xBF))) {
            return true;
        } else if (((code1 >= 0xF1) && (code1 <= 0xF3))
                   && ((code2 >= 0x80) && (code2 <= 0xBF))
                   && ((code3 >= 0x80) && (code3 <= 0xBF))
                   && ((code4 >= 0x80) && (code4 <= 0xBF))) {
            return true;
        } else if ((0xF4 == code1)
                   && ((code2 >= 0x80) && (code2 <= 0x8F))
                   && ((code3 >= 0x80) && (code3 <= 0xBF))
                   && ((code4 >= 0x80) && (code4 <= 0xBF))) {
            return true;
        }
    }

    return false;
}

int main (int argc, char* argv[])
{
    // utf-8 1 - 4 字节
    const char* arr[] = {
            " ", "/", "0", "9", ":", "@", "A", "Z", "[", "`", "a", "z", "{", "~",       /* 以下都是 ASCII */
            "§", "©", "®", "Ą", "Ď", "Đ", "Ő",                                          /* 2 字节 utf8 编码 */
            "ᥕ", "一", "龥", "庄", "丽", "君",                                            /* 3 字节 */
            "契", "い", "龜"
    };

    printf ("array: %lu/%lu\n\n", sizeof arr[0], sizeof arr);

    char buf[128] = {0};
    int len = sizeof arr / sizeof arr[0];
    for (int i = 0; i < len; ++i) {
        const char* code = arr[i];

        if (is_utf8_1 (code)) {
            printf ("index: %-2d --> utf8-1 code: '%c'  --  hex code: 0x%-02X <==> dec: %d\n", i, code[0], code[0] & 0xFF, code[0] & 0xFF);
            continue;
        } else if (is_utf8_2 (code)) {
            printf ("index: %-2d --> utf8-2 code: '%s'  --  hex code: 0x%-02X 0x%-02X\n", i, code, code[0] & 0xFF, code[1] & 0xFF);
            continue;
        } else if (is_utf8_3 (code)) {
            printf ("index: %-2d --> utf8-3 code: '%s'  --  hex code: 0x%-02X 0x%-02X 0x%-02X\n", i, code, code[0] & 0xFF, code[1] & 0xFF, code[2] & 0xFF);
        } else if (is_utf8_4 (code)) {
            printf ("index: %-2d --> utf8-4 code: '%s'  --  hex code: 0x%-02X 0x%-02X 0x%-02X 0x%-02X\n", i, code, code[0] & 0xFF, code[1] & 0xFF, code[2] & 0xFF, code[3] & 0xFF);
        }
    }

    return 0;
}

1

gcc -O0 -finput-charset=utf8 -fexec-charset=utf8 demo-utf8.run demo-utf8.c