0
Good evening, everyone!
I am making a system in C# Windowsforms for validation of phone bills model FEBRABAN V2.
The files usually have between 600Mb to 15Gb, for this reason I opted for the use of Parallel.Foreach, and really transformed the performance in a gigantic gain, things from 2h of validation fell to mere 40s in reading almost 5 million lines.
Going to the system.
//MAIN CLASS OF THE SYSTEM
try
{
int? linhaTotal = File.ReadAllLines(catalogPath).Count();
Parallel.ForEach(File.ReadLines(catalogPath), (linha, pls, rownumber) =>
{
if (linha.Length == 350)
{
String line = linha.Substring(0, 1);
switch (line)
{
case "0": logicaV2.ValidaHeader(Convert.ToInt32(rownumber + 1), linha, header); break;
case "1": logicaV2.ValidaContaResumo(Convert.ToInt32(rownumber + 1), linha, contaResumo); break;
case "2": logicaV2.ValidaEndereco(Convert.ToInt32(rownumber + 1), linha, enderecoPonta); break;
case "3": retornoGeral = logicaV2.ValidaBilhetacao(Convert.ToInt32(rownumber + 1), linha, bilhetacao, listaErro, pkArquivo); break;
//case "4": logicaV2.ValidaServico(sequencial, linha); break;
//case "5": logicaV2.ValidaDescontos(sequencial, linha); break;
//case "9": logicaV2.ValidaTrailler(sequencial, linha); break;
}
}
else
{
Console.WriteLine("Linha: {0}", rownumber);
}
//valorTotalFatura = retornoGeral.listaValor.Sum();
});
}
catch (AggregateException e)
{
Console.WriteLine("Erro Geral: {0} ", e);
}
//CLASS Nevalidadorv2 (where validations of each type are made, I only posted one of the 9 as an example)
V2DB001DataContext context = new V2DB001DataContext();
Boolean isValid = true;
Form1.objErros objErros = new Form1.objErros();
Form1.objRetorno retornoGeral = new Form1.objRetorno();
public decimal? valorLigacao = 0,
valorTotal = 0;
#region VALIDA BILHETAÇÃO
public Form1.objRetorno ValidaBilhetacao(int sequencial, String linha, List<T004_BILHETACAO> bilhetacao, List<T009_ERRO> listaErro, int? pkArquivo)
{
foreach (var item in bilhetacao)
{
String campo = linha.Substring((int)item.T004_POSICAO_INICIO, (int)item.T004_POSICAO_FIM);
if (item.T004_DE_REGISTRO.Equals("TIPO_REGISTRO"))
{
isValid = (campo.Length == item.T004_POSICAO_FIM ? (campo.Equals("3") ? true : false) : false);
}
else if (item.T004_DE_REGISTRO.Equals("SEQUENCIAL_GRAVACAO"))
{
isValid = (campo.Length == item.T004_POSICAO_FIM ? (int.Parse(campo).Equals(sequencial) ? true : false) : false);
}
else if (item.T004_DE_REGISTRO.Equals("DT_VENCIMENTO"))
{
if (campo.Length == item.T004_POSICAO_FIM)
{
try
{
campo = campo.Substring(0, 4) + "-" + campo.Substring(4, 2) + "-" + campo.Substring(6, 2);
Convert.ToDateTime(campo);
isValid = true;
}
catch
{
isValid = false;
}
}
}
else if (item.T004_DE_REGISTRO.Equals("DT_EMISSAO"))
{
if (campo.Length == item.T004_POSICAO_FIM)
{
try
{
campo = campo.Substring(0, 4) + "-" + campo.Substring(4, 2) + "-" + campo.Substring(6, 2);
Convert.ToDateTime(campo);
isValid = true;
}
catch
{
isValid = false;
}
}
}
else if (item.T004_DE_REGISTRO.Equals("ID_UNICO_RECURSO"))
{
if (campo.Length == item.T004_POSICAO_FIM)
{
if (campo.Trim().Length <= item.T004_POSICAO_FIM)
{
isValid = true;
}
else
{
isValid = false;
}
}
else
{
isValid = false;
}
}
else if (item.T004_DE_REGISTRO.Equals("CNL_RECURSO_REFERENCIA"))
{
isValid = (campo.Length == item.T004_POSICAO_FIM ? true : false);
}
else if (item.T004_DE_REGISTRO.Equals("DDD"))
{
isValid = (campo.Length == item.T004_POSICAO_FIM ? validaDDD(campo) : false);
}
else if (item.T004_DE_REGISTRO.Equals("NU_TELEFONE"))
{
isValid = (campo.Length == item.T004_POSICAO_FIM ? true : false);
}
else if (item.T004_DE_REGISTRO.Equals("CARACTERISTICA_RECURSO"))
{
isValid = (campo.Length == item.T004_POSICAO_FIM ? true : false);
}
else if (item.T004_DE_REGISTRO.Equals("DEGRAU_RECURSO"))
{
isValid = (campo.Length == item.T004_POSICAO_FIM ? true : false);
}
else if (item.T004_DE_REGISTRO.Equals("DT_LIGACAO"))
{
if (campo.Length == item.T004_POSICAO_FIM)
{
try
{
campo = campo.Substring(0, 4) + "-" + campo.Substring(4, 2) + "-" + campo.Substring(6, 2);
Convert.ToDateTime(campo);
isValid = true;
}
catch
{
isValid = false;
}
}
}
else if (item.T004_DE_REGISTRO.Equals("CNL_LOCALIDADE_CHAMADA"))
{
isValid = (campo.Length == item.T004_POSICAO_FIM ? true : false);
}
else if (item.T004_DE_REGISTRO.Equals("NO_LOCALIDADE_CHAMADA"))
{
if (campo.Length == item.T004_POSICAO_FIM)
{
if (campo.Trim().Length <= item.T004_POSICAO_FIM)
{
isValid = true;
}
else
{
isValid = false;
}
}
else
{
isValid = false;
}
}
else if (item.T004_DE_REGISTRO.Equals("UF_FONE_CHAMADO"))
{
isValid = (campo.Length == item.T004_POSICAO_FIM ? validaUf(campo) : false);
}
else if (item.T004_DE_REGISTRO.Equals("CO_NACIONAL_INTERNACIONAL"))
{
isValid = (campo.Length == item.T004_POSICAO_FIM ? (campo.Trim().Equals("0") || campo.Trim().Equals("00") ? true : false) : false);
}
else if (item.T004_DE_REGISTRO.Equals("CO_OPERADORA"))
{
isValid = (campo.Length == item.T004_POSICAO_FIM ? validaCodOperadora(campo) : false);
}
else if (item.T004_DE_REGISTRO.Equals("DE_OPERADORA"))
{
if (campo.Length == item.T004_POSICAO_FIM)
{
if (campo.Trim().Length <= item.T004_POSICAO_FIM)
{
isValid = true;
}
else
{
isValid = false;
}
}
else
{
isValid = false;
}
}
else if (item.T004_DE_REGISTRO.Equals("CO_PAIS_CHAMADO"))
{
isValid = (campo.Length == item.T004_POSICAO_FIM ? String.IsNullOrEmpty(campo.Trim()) ? true : validaCodInternacional(campo) : false);
}
else if (item.T004_DE_REGISTRO.Equals("AREA_DDD"))
{
isValid = (campo.Length == item.T004_POSICAO_FIM ? String.IsNullOrEmpty(campo.Trim()) ? true : validaDDD(campo.Trim()) : false);
}
else if (item.T004_DE_REGISTRO.Equals("NU_FONE_CHAMADO"))
{
isValid = (campo.Length == item.T004_POSICAO_FIM ? true : false);
}
else if (item.T004_DE_REGISTRO.Equals("CONJUGADO_CHAMADO"))
{
if (campo.Length == item.T004_POSICAO_FIM)
{
if (campo.Trim().Length <= item.T004_POSICAO_FIM)
{
isValid = true;
}
else
{
isValid = false;
}
}
else
{
isValid = false;
}
}
else if (item.T004_DE_REGISTRO.Equals("DURACAO"))
{
isValid = (campo.Length == item.T004_POSICAO_FIM ? true : false);
}
else if (item.T004_DE_REGISTRO.Equals("CATEGORIA"))
{
isValid = (campo.Length == item.T004_POSICAO_FIM ? true : false);
}
else if (item.T004_DE_REGISTRO.Equals("DE_CATEGORIA"))
{
if (campo.Length == item.T004_POSICAO_FIM)
{
if (campo.Trim().Length <= item.T004_POSICAO_FIM)
{
isValid = true;
}
else
{
isValid = false;
}
}
else
{
isValid = false;
}
}
else if (item.T004_DE_REGISTRO.Equals("HORARIO_LIGACAO"))
{
isValid = (campo.Length == item.T004_POSICAO_FIM ? true : false);
}
else if (item.T004_DE_REGISTRO.Equals("TIPO_CHAMADA"))
{
isValid = (campo.Length == item.T004_POSICAO_FIM ? true : false);
}
else if (item.T004_DE_REGISTRO.Equals("GRUPO_HOR_TARIFARIO"))
{
isValid = (campo.Length == item.T004_POSICAO_FIM ? true : false);
}
else if (item.T004_DE_REGISTRO.Equals("DE_HOR_TARIFARIO"))
{
isValid = (campo.Length == item.T004_POSICAO_FIM ? true : false);
}
else if (item.T004_DE_REGISTRO.Equals("DEGRAU_LIGACAO"))
{
isValid = (campo.Length == item.T004_POSICAO_FIM ? (!String.IsNullOrEmpty(campo.Trim()) ? true : false) : false);
}
else if (item.T004_DE_REGISTRO.Equals("SINAL_VALOR_LIGACAO"))
{
isValid = (campo.Length == item.T004_POSICAO_FIM ? (campo.Equals("+") || campo.Equals("-") ? true : false) : false);
}
else if (item.T004_DE_REGISTRO.Equals("ALIQUOTA"))
{
isValid = (campo.Length == item.T004_POSICAO_FIM ? true : false);
}
else if (item.T004_DE_REGISTRO.Equals("VALOR_LIGACAO"))
{
if (campo.Length == item.T004_POSICAO_FIM)
{
if (campo.Trim().Length <= item.T004_POSICAO_FIM)
{
isValid = true;
valorLigacao = (decimal?)Convert.ToInt32(campo) / 100;
valorTotal += valorLigacao;
//Console.WriteLine("Linha: {0} Valor: {1} Total: {2}", sequencial, valorLigacao, valorTotal);
retornoGeral.listaValor.Add(valorLigacao);
}
else
{
isValid = false;
}
}
else
{
isValid = false;
}
}
else if (item.T004_DE_REGISTRO.Equals("CLASSE_SERVICO"))
{
isValid = (campo.Length == item.T004_POSICAO_FIM ? true : false);
}
else if (item.T004_DE_REGISTRO.Equals("FILLER"))
{
isValid = (campo.Length == item.T004_POSICAO_FIM ? ( String.IsNullOrEmpty(campo.Trim()) ? true : false) : false);
}
if (!isValid)
{
bool valida = armazenaErro(listaErro, sequencial, pkArquivo, item.T004_OBSERVACAO, (bool)item.T004_FLAG_IMPEDITIVO);
isValid = true;
}
}
retornoGeral.objRetornoErros = objErros;
retornoGeral.valorTotal = valorTotal;
return retornoGeral;
}
#endregion
But briefly, the process is divided between my two nuclei... The problem is that one kernel can arrive at the end of the file while the other is somewhere else in the file, so it cannot return me the EXACT FINAL VALUE, it always returns me where the last Thread is finished.
I wonder if there is any way to perfectly synchronize the Threads making my return is the last given being the last line of my document.
Who threads? I didn’t see any in the code.
– Maniero
As far as I know, parallelism divides the process between the processors, if I mistake the term I’m sorry.
– Leonardo Amaral
No, nothing guarantees that there will be this separation. As access is done on the disk, the gain comes from the fact that it does not wait for the load. The code has several small things that could be improved, but nothing important (it has a lot of redundancy and things that are simply not being used. A good revision of the code would help a lot. I don’t know what the doubt is anyway. It is to have a big gain, but of 2hs p/ 40s indicates that there is something wrong either in the previous algorithm or in this. Just parallelizing can’t earn that much.
– Maniero
In fact the code passed to this current one I effected several changes, but only had a nice gain after using Parallelism. At the end I have to check if the total value of the ticketing is equal to the last line of the TXT, but my list is returning random lines, for example, my test file has 31115 lines, instead of returning me this total it returns me 31112, 31109 and never the last line, I tried to run with TASK and Task.Whenall() and likewise continues to bring me this way, I would like you to return me the ULTIMA LINE. XD
– Leonardo Amaral
There is a lot of redundancy, because I did not totally separate the project into classes and also several fields to be validated separately.
– Leonardo Amaral
Redundancies are much more basic than this.
– Maniero