C# Hadoop学习笔记（一）—环境安装

时间：2017-01-12 14:23:20 阅读：580 评论：0 收藏：0 [点我收藏+]

一、安装环境

1，前期准备：官网下载“NuGet Package Manager”，按自己已有的VS环境下载对应版本；

2，利用NuGet下载Hadoop For .NET SDK,地址“http://hadoopsdk.codeplex.com/”

3，安装。

4，通过HDInsight，安装Windows Azure，目前是预览版本。

5，参照网址“http://blogs.msdn.com/b/data_otaku/archive/2013/08/14/hadoop-for-net-developers.aspx” 系统学习API

二、测试DEMO

[csharp] view plain copy

using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using Microsoft.Hadoop;
using Microsoft.Hadoop.MapReduce;
using Microsoft.Hadoop.WebClient.WebHCatClient;
using System.Diagnostics;
using System.IO;
using System.IO.MemoryMappedFiles;
namespace HadoopConsol
{
class Program
{
static void Main(string[] args)
{
Stopwatch sw = new Stopwatch();
long hadoopTime=0;
long normalTime=0;
sw.Start();
//start hadoop
Console.WriteLine(" Hadoop Process Strating ....");
#region Hadoop time
#region hadoopconnet
Console.WriteLine(" Hadoop Connect Strating ....");
//establish job configuration
HadoopJobConfiguration myConfig = new HadoopJobConfiguration();
myConfig.InputPath = "/demo/simple/in";
myConfig.OutputFolder = "/demo/simple/out";
//connect to cluster
Uri myUri = new Uri("http://localhost");
string userName = "hadoop";
string passWord = null;
IHadoop myCluster = Hadoop.Connect(myUri, userName, passWord);
hadoopTime += sw.ElapsedMilliseconds;
Console.WriteLine(" Hadoop Connect End.");
Console.WriteLine(" Hadoop Connect time:" + sw.ElapsedMilliseconds);
#endregion
#region hadoopmapreduce
sw.Reset();
sw.Start();
Console.WriteLine(" Hadoop MapReduce Strating ....");
//execute mapreduce job
MapReduceResult jobResult =
myCluster.MapReduceJob.Execute<MySimpleMapper, MySimpleReducer>(myConfig);
hadoopTime += sw.ElapsedMilliseconds;
Console.WriteLine(" Hadoop MapReduce End.");
Console.WriteLine(" Hadoop MapReduce Time:"+sw.ElapsedMilliseconds);
#endregion
#region Hadoop End
sw.Reset();
sw.Start();
Console.WriteLine(" Hadoop Endprocess Strating ....");
//write job result to console
int exitCode = jobResult.Info.ExitCode;
string exitStatus = "Failure";
if (exitCode == 0) exitStatus = "Success";
exitStatus = exitCode + " (" + exitStatus + ")";
Console.WriteLine();
Console.Write("Exit Code = " + exitStatus);
Console.WriteLine(" Hadoop Endprocess End.");
hadoopTime += sw.ElapsedMilliseconds;
Console.WriteLine(" Hadoop Exit Time:" + sw.ElapsedMilliseconds);
Console.WriteLine(" Hadoop Process All Time:" + hadoopTime);
#endregion
#endregion
#region Normal time
//start Normal
Console.WriteLine(" Normal Process Strating ....");
sw.Reset();
sw.Start();
//normal process
#region Normal Process
int myevenCount = 0;
int myeventSum = 0;
int myoddCount = 0;
int myoddSum = 0;
StreamReader fs = new StreamReader(@"c:\TEMP\integers.txt");
while (fs.Peek() >= 0)
{
string strTemp = fs.ReadLine();
if (Int32.Parse(strTemp) % 2 == 0)
{
myevenCount++;
myeventSum += Int32.Parse(strTemp);
}
else
{
myoddCount++;
myoddSum += Int32.Parse(strTemp);
}
}
//MemoryMappedFile m = MemoryMappedFile.
Console.WriteLine("even:" + "\t" + myevenCount + "\t" + myeventSum);
Console.WriteLine("odd:" + "\t" + myoddCount + "\t" + myoddSum);
#endregion
Console.WriteLine(" Normal Process End.");
normalTime += sw.ElapsedMilliseconds;
Console.WriteLine(" Normal Exit Time:" + sw.ElapsedMilliseconds);
Console.WriteLine(" Normal Process All Time:" + normalTime);
#endregion
sw.Stop();
Console.Read();
}
}
public class MySimpleMapper : MapperBase
{
public override void Map(string inputLine, MapperContext context)
{
//interpret the incoming line as an integer value
int value = int.Parse(inputLine);
//determine whether value is even or odd
string key = (value % 2 == 0) ? "even" : "odd";
//output key assignment with value
context.EmitKeyValue(key, value.ToString());
}
}
public class MySimpleReducer : ReducerCombinerBase
{
public override void Reduce(
string key, IEnumerable<string> values, ReducerCombinerContext context
)
{
//initialize counters
int myCount = 0;
int mySum = 0;
//count and sum incoming values
foreach (string value in values)
{
mySum += int.Parse(value);
myCount++;
}
//output results
context.EmitKeyValue(key, myCount + "\t" + mySum);
}
}
}

三、测试结果

C# Hadoop学习笔记（一）—环境安装

原文：http://www.cnblogs.com/jjg0519/p/6278020.html

踩

(0)

评论一句话评论（0）

分享档案

更多>

2021年09月23日 (328)
2021年09月24日 (313)
2021年09月17日 (191)
2021年09月15日 (369)
2021年09月16日 (411)
2021年09月13日 (439)
2021年09月11日 (398)
2021年09月12日 (393)
2021年09月10日 (160)
2021年09月08日 (222)