Man I went through this problem and decided to dedicate myself to creating a directive that solves me the problem and as you can see below, she got big.
basically you put as attribute of your input a number-only and ready, by default it only accepts whole numbers and positive.
If you want decimal numbers you put number-only="." or number-only="," she accepts , and . as decimal separator to display in the view.
If you want negative numbers you put number-only="-" and ready, you can insert the - in front of the number.
If you want to accept any numbers number-only="-." number-only="-,", it will accept decimal and negative numbers plus positive integers.
Attention! It does not accept thousand separator, like 1.000,00, only symbol to separate decimal integers, and only the symbol you chose in the directive statement( , or . )
.directive('numberOnly', function() {
  return {
    restrict: 'A',
    require: 'ngModel',
    link: function(scope, element, attrs, ngModel) {
      var negativo = /\-/.test(attrs.numberOnly);
      var decimal = /\.|\,/.test(attrs.numberOnly) ? /\.|\,/.exec(attrs.numberOnly)[0] : null;
      var regExp = '^';
      regExp += negativo ? '[\\-]{0,1}' : '';
      regExp += '[\\d]+';
      if(decimal != null) {
        regExp += '[\\'+decimal+'][\\d]+|';
        if(negativo) {
          regExp += '[\\-]{0,1}'
        }
        regExp += '[\\d]+'
      }
      regExp += '';
      regExp = new RegExp(regExp);
      ngModel.$parsers.unshift(function(input) {
        if(input === undefined) return null;
        if(input === null) return;
        input = input.toString().replace(/\./, decimal);
        if(input == '-') return input;
        if(decimal !== null && input.charAt(input.length-1) == decimal) return input;
        input = regExp.test(input) ? regExp.exec(input)[0] : null;
        var viewVal = null;
        if (input !== null) {
          input = decimal != null ? parseFloat(input.replace(/\,/, '.')) : parseInt(input);
        }
        viewVal = isNaN(input) || input === null ? '' : input;
        ngModel.$setViewValue(decimal != null ? viewVal.toString().replace(/\./, decimal) : viewVal.toString());
        ngModel.$render();
        return isNaN(input) ? null : input;
      });
      ngModel.$formatters.unshift(function(value) {
        if(value !== undefined && value !== null) {
          return decimal != null ? value.toString().replace(/\./, decimal) : value.toString();
        }
      });
    }
  }
});
HTML
<input ng-model="var1" number-only />        // apenas inteiros positivos
<input ng-model="var2" number-only="." />    // apenas decimais positivos, separados por ponto
<input ng-model="var3" number-only="," />    // apenas decimais positivos, separados por vírgula
<input ng-model="var4" number-only="-" />    // apenas números inteiros positivos ou negativos
<input ng-model="var5" number-only=",-" />   // quaisquer números inteiros ou decimais, positivos ou negativos separados por vírgula
<input ng-model="var6" number-only="-." />   // quaisquer números inteiros ou decimais, positivos ou negativos separados por ponto
							
							
						 
Have you tried ng-Pattern?
– DiegoAugusto
@Diegoaugusto tried, continue allowing informs alpha characters.
– Oliveira
Allow will continue even, but will give error in the form when trying to do Submit. If you want to block number entry, you will need a directive, e.g.: https://hassantariqblog.wordpress.com/2015/04/25/angularjs-directive-to-allow-number-only-in-textbox/
– DiegoAugusto