しっぽを追いかけて

ぐるぐるしながら考えています

Unity と猫の話題が中心   掲載内容は個人の私見であり、所属組織の見解ではありません

手書き文字を認識して読み上げてみる

手書き文字の認識率を確かめるために書いた文字を読み上げるサンプルを作ってみました!

f:id:matatabi_ux:20140508221553p:plain

いつぞやのお絵かき Canvas を流用して文字認識機能をつけてみます

/// <summary>
/// 文字認識イベント引数
/// </summary>
public class RecognizedEventArgs : EventArgs
{
    /// <summary>
    /// 認識文字列
    /// </summary>
    public string Message { get; private set; }

    /// <summary>
    /// コンストラクタ
    /// </summary>
    public RecognizedEventArgs(string message)
    {
        this.Message = message;
    }
}

/// <summary>
/// 手書き文字認識できる Canvas
/// </summary>
public class RecognizableCanvas : Canvas
{
    #region Privates

    /// <summary>
    /// 描画管理クラス
    /// </summary>
    private InkManager inkManager = new InkManager();

    /// <summary>
    /// 手書き認識クラス
    /// </summary>
    private InkRecognizer inkRecognizer;

    /// <summary>
    /// 手書き認識間隔タイマー
    /// </summary>
    private DispatcherTimer timer = new DispatcherTimer();

    /// <summary>
    /// ストロークID
    /// </summary>
    private uint penId = 0;

    /// <summary>
    /// ポインタ座標
    /// </summary>
    private Point point = new Point();

    /// <summary>
    /// 描画中のペン色
    /// </summary>
    private Color penColor = Colors.Black;

    /// <summary>
    /// 描画ストローク
    /// </summary>
    private IList<IList<Shape>> strokes = new List<IList<Shape>>();

    /// <summary>
    /// 描画ストロークスタック
    /// </summary>
    private IList<IList<Shape>> strokeStack = new List<IList<Shape>>();

    /// <summary>
    /// 描画線
    /// </summary>
    private IList<Shape> lines = new List<Shape>();

    /// <summary>
    /// StrokeColor 依存関係プロパティ
    /// </summary>
    private readonly DependencyProperty StrokeColorProperty
        = DependencyProperty.Register("StrokeColor", typeof(Color), typeof(RecognizableCanvas), new PropertyMetadata(Colors.White));

    /// <summary>
    /// StrokeWidth 依存関係プロパティ
    /// </summary>
    private readonly DependencyProperty StrokeWidthProperty
        = DependencyProperty.Register("StrokeWidth", typeof(double), typeof(RecognizableCanvas), new PropertyMetadata(8d));

    /// <summary>
    /// CanUndo 依存関係プロパティ
    /// </summary>
    private readonly DependencyProperty CanUndoProperty
        = DependencyProperty.Register("CanUndo", typeof(bool), typeof(RecognizableCanvas), new PropertyMetadata(false));

    /// <summary>
    /// CanRedo 依存関係プロパティ
    /// </summary>
    private readonly DependencyProperty CanRedoProperty
        = DependencyProperty.Register("CanRedo", typeof(bool), typeof(RecognizableCanvas), new PropertyMetadata(false));

    /// <summary>
    /// デフォルトの描画設定
    /// </summary>
    private InkDrawingAttributes drawingAttributes = new InkDrawingAttributes()
    {
        Color = Colors.White,
        FitToCurve = true,
        IgnorePressure = false,
        PenTip = PenTipShape.Circle,
        Size = new Size(8, 8),
    };

    #endregion //Privates

    /// <summary>
    /// 文字認識イベントハンドラ
    /// </summary>
    /// <param name="e">イベント引数</param>
    public delegate void RecognizedEventHandler(object sender, RecognizedEventArgs e);

    /// <summary>
    /// 文字認識イベント
    /// </summary>
    public event RecognizedEventHandler Recognized;

    /// <summary>
    /// コンストラクタ
    /// </summary>
    public RecognizableCanvas()
    {
        this.Loaded += this.OnLoaded;
        this.Unloaded += this.OnUnloaded;

        this.timer.Interval = TimeSpan.FromMilliseconds(500);
        this.inkRecognizer = this.inkManager.GetRecognizers().FirstOrDefault(r => r.Name.Contains("日本語"));
        if (this.inkRecognizer == null)
        {
            return;
        }
        this.inkManager.SetDefaultRecognizer(this.inkRecognizer);
    }

    /// <summary>
    /// StrokeColor プロパティ
    /// </summary>
    public Color StrokeColor
    {
        get { return (Color)this.GetValue(this.StrokeColorProperty); }
        set { this.SetValue(this.StrokeColorProperty, value); }
    }

    /// <summary>
    /// StrokeWidth プロパティ
    /// </summary>
    public double StrokeWidth
    {
        get { return (double)this.GetValue(this.StrokeWidthProperty); }
        set { this.SetValue(this.StrokeWidthProperty, value); }
    }

    /// <summary>
    /// CanUndo プロパティ
    /// </summary>
    public bool CanUndo
    {
        get { return (bool)this.GetValue(this.CanUndoProperty); }
        set { this.SetValue(this.CanUndoProperty, value); }
    }

    /// <summary>
    /// CanRedo プロパティ
    /// </summary>
    public bool CanRedo
    {
        get { return (bool)this.GetValue(this.CanRedoProperty); }
        set { this.SetValue(this.CanRedoProperty, value); }
    }

    /// <summary>
    /// 読み込み完了イベントハンドラ
    /// </summary>
    /// <param name="sender">イベント発行者</param>
    /// <param name="e">イベント引数</param>
    private void OnLoaded(object sender, Windows.UI.Xaml.RoutedEventArgs e)
    {
        this.Loaded -= this.OnLoaded;

        this.timer.Tick += this.OnTimeout;
        this.PointerPressed += this.OnPointerPressed;
        this.PointerMoved += this.OnPointerMoved;
        this.PointerExited += this.OnPointerReleased;
        this.PointerReleased += this.OnPointerReleased;
    }

    /// <summary>
    /// 破棄完了イベントハンドラ
    /// </summary>
    /// <param name="sender">イベント発行者</param>
    /// <param name="e">イベント引数</param>
    private void OnUnloaded(object sender, Windows.UI.Xaml.RoutedEventArgs e)
    {
        this.Unloaded -= this.OnUnloaded;
        this.Loaded -= this.OnLoaded;

        this.timer.Tick -= this.OnTimeout;
        this.PointerPressed -= this.OnPointerPressed;
        this.PointerMoved -= this.OnPointerMoved;
        this.PointerExited -= this.OnPointerReleased;
        this.PointerReleased -= this.OnPointerReleased;
    }

    /// <summary>
    /// 押下イベントハンドラ
    /// </summary>
    /// <param name="sender">イベント発行者</param>
    /// <param name="e">イベント引数</param>
    private void OnPointerPressed(object sender, Windows.UI.Xaml.Input.PointerRoutedEventArgs e)
    {
        try
        {
            if (penId == 0)
            {
                this.timer.Stop();
                this.lines = new List<Shape>();
                var pointer = e.GetCurrentPoint(this);
                this.penColor = this.StrokeColor;
                this.drawingAttributes.Color = this.StrokeColor;
                this.inkManager.SetDefaultDrawingAttributes(drawingAttributes);
                if (pointer.Properties.IsLeftButtonPressed)
                {
                    this.inkManager.ProcessPointerDown(pointer);
                    this.penId = pointer.PointerId;
                    this.point = pointer.Position;
                    e.Handled = true;
                }
            }
        }
        catch (Exception)
        {
        }
    }

    /// <summary>
    /// ドラッグイベントハンドラ
    /// </summary>
    /// <param name="sender">イベント発行者</param>
    /// <param name="e">イベント引数</param>
    private void OnPointerMoved(object sender, Windows.UI.Xaml.Input.PointerRoutedEventArgs e)
    {
        if (e.Pointer.PointerId.Equals(this.penId))
        {
            var pointer = e.GetCurrentPoint(this);

            this.DrawLine(
                this.point.X,
                this.point.Y,
                pointer.Position.X,
                pointer.Position.Y,
                this.StrokeWidth,
                new SolidColorBrush(this.penColor)
            );
            this.inkManager.ProcessPointerUpdate(pointer);
            this.point = pointer.Position;
        }
    }

    /// <summary>
    /// 押上イベントハンドラ
    /// </summary>
    /// <param name="sender">イベント発行者</param>
    /// <param name="e">イベント引数</param>
    private void OnPointerReleased(object sender, Windows.UI.Xaml.Input.PointerRoutedEventArgs e)
    {
        if (e.Pointer.PointerId.Equals(this.penId))
        {
            var pointer = e.GetCurrentPoint(this);
            this.timer.Start();

            this.penId = 0;
            this.inkManager.ProcessPointerUp(pointer);
            this.strokes.Add(this.lines);
            this.CanRedo = false;
            this.strokeStack.Clear();
            this.CanUndo = true;
        }
    }

    /// <summary>
    /// 線を描く
    /// </summary>
    /// <param name="x1">始点 x 座標</param>
    /// <param name="y1">始点 y 座標</param>
    /// <param name="x2">終点 x 座標</param>
    /// <param name="y2">終点 y 座標</param>
    /// <param name="thickness">太さ</param>
    /// <param name="stroke"></param>
    /// <param name="invoke">イベント通知</param>
    private void DrawLine(double x1, double y1, double x2, double y2, double thickness, SolidColorBrush stroke, bool invoke = true)
    {
        var line = new Line()
        {
            X1 = x1,
            Y1 = y1,
            X2 = x2,
            Y2 = y2,
            StrokeThickness = thickness,
            Stroke = stroke,
            StrokeLineJoin = PenLineJoin.Round,
            StrokeStartLineCap = PenLineCap.Round,
            StrokeEndLineCap = PenLineCap.Round,
        };
        this.Children.Add(line);
        this.lines.Add(line);
    }

    /// <summary>
    /// タイムアウトイベントハンドラ
    /// </summary>
    /// <param name="sender">イベント発行者</param>
    /// <param name="e">イベント引数</param>
    private async void OnTimeout(object sender, object e)
    {
        this.timer.Stop();
        try
        {
            var recognized = (await this.inkManager.RecognizeAsync(InkRecognitionTarget.All)).FirstOrDefault();
            if (recognized == null)
            {
                return;
            }
            if (this.Recognized == null)
            {
                return;
            }
            this.Recognized(this, new RecognizedEventArgs(recognized.GetTextCandidates().FirstOrDefault()));
            return;
        }
        catch(Exception ex)
        {
            Debug.WriteLine(ex.ToString());
        }
        await this.ClearAsync();
    }

    /// <summary>
    /// 元に戻す
    /// </summary>
    /// <returns>Task</returns>
    public void Undo()
    {
        var last = this.strokes.LastOrDefault();
        if (last != null)
        {
            this.strokeStack.Add(last);
            this.strokes.Remove(last);
            this.CanRedo = true;
            foreach (var line in last)
            {
                this.Children.Remove(line);
            }
        }
        if (this.strokes.Count <= 0)
        {
            this.CanUndo = false;
        }
    }

    /// <summary>
    /// やり直し
    /// </summary>
    /// <returns>Task</returns>
    public void Redo()
    {
        var last = this.strokeStack.LastOrDefault();
        if (last != null)
        {
            this.strokes.Add(last);
            this.strokeStack.Remove(last);
            this.CanUndo = true;
            foreach (var line in last)
            {
                this.Children.Add(line);
            }
            if (this.strokeStack.Count <= 0)
            {
                this.CanRedo = false;
            }
        }
    }

    /// <summary>
    /// 全消しする
    /// </summary>
    /// <returns>Task</returns>
    public async Task ClearAsync()
    {
        await this.Dispatcher.RunAsync(CoreDispatcherPriority.Normal, () =>
        {
            foreach (var stroke in this.inkManager.GetStrokes())
            {
                stroke.Selected = true;
            }
            this.inkManager.DeleteSelected();

            this.penId = 0;
            this.CanUndo = false;
            this.CanRedo = false;
            this.strokes.Clear();
            this.strokeStack.Clear();
            this.lines.Clear();
            this.Children.Clear();
        });
    }
}

xaml にはこんな感じで配置して

<ctrl:RecognizableCanvas x:Name="RecognizableCanvas"
                         Grid.RowSpan="2"
                         Background="Transparent"
                         Recognized="OnRecognized" />

コードビハインドなどで認識した文字を音声合成で読み上げてみます

/// <summary>
/// メディア再生
/// </summary>
private MediaElement mediaElement = new MediaElement();

/// <summary>
/// 文字認識イベントハンドラ
/// </summary>
/// <param name="sender">イベント発行者</param>
/// <param name="e">イベント引数</param>
private async void OnRecognized(object sender, RecognizedEventArgs e)
{
    try
    {
        // 認識した文字を音声合成で読み上げる
        using (var synth = new SpeechSynthesizer())
        {
            var stream = await synth.SynthesizeTextToStreamAsync(e.Message);
            this.mediaElement.SetSource(stream, stream.ContentType);
            this.mediaElement.Play();
        }
    }
    catch(Exception ex)
    {
        Debug.WriteLine(ex.ToString());
    }
    await this.RecognizableCanvas.ClearAsync();
}

漢字はへんとつくりが分かれて認識されるみたいでいまいちでしたが、結構精度は高くて何かの入力補助で使いたいなと思うほどでした!