opening a gzip compressed file from hive is returning "Not a data file" - amazon-s3

below is my configuration which works perfectly fine for non compressed data
agent.sinks.test.type = hdfs
agent.sinks.test.hdfs.useLocalTimeStamp = true
agent.sinks.test.hdfs.path = s3n://AccessKeys#test/%{topic}/utc=%s
agent.sinks.test.hdfs.roundUnit = minute
agent.sinks.test.hdfs.round = true
agent.sinks.test.hdfs.roundValue = 10
agent.sinks.test.hdfs.fileSuffix = .avro
agent.sinks.test.serializer =
com.test.flume.sink.serializer.GenericRecordAvroEventSerializer$Builder
agent.sinks.test.hdfs.fileType = DataStream
agent.sinks.test.hdfs.maxOpenFiles=100
agent.sinks.test.hdfs.appendTimeout = 5000
agent.sinks.test.hdfs.callTimeout = 4000
agent.sinks.test.hdfs.rollInterval = 60
agent.sinks.test.hdfs.rollSize = 0
agent.sinks.test.hdfs.rollCount = 1000
agent.sinks.test.hdfs.batchSize = 1000
agent.sinks.test.hdfs.threadsPoolSize=100
I am trying to add compression using gzip to this as follows
agent.sinks.test.type = hdfs
agent.sinks.test.hdfs.useLocalTimeStamp = true
agent.sinks.test.hdfs.path = s3n://AccessKeys#test/%{topic}/utc=%s
agent.sinks.test.hdfs.roundUnit = minute
agent.sinks.test.hdfs.round = true
agent.sinks.test.hdfs.roundValue = 10
agent.sinks.test.hdfs.fileSuffix = .avro
agent.sinks.test.serializer =
com.test.flume.sink.serializer.GenericRecordAvroEventSerializer$Builder
agent.sinks.test.hdfs.fileType = CompressedStream
agent.sinks.test.hdfs.codeC = gzip
agent.sinks.test.hdfs.maxOpenFiles=100
agent.sinks.test.hdfs.appendTimeout = 10000
agent.sinks.test.hdfs.callTimeout = 4000
agent.sinks.test.hdfs.rollInterval = 60
agent.sinks.test.hdfs.rollSize = 0
agent.sinks.test.hdfs.rollCount = 1000
agent.sinks.test.hdfs.batchSize = 1000
agent.sinks.test.hdfs.threadsPoolSize=100
All the above data is being stored in s3 and when i try to retrieve the data from hive i am getting the below error.
Exception in thread "main" java.io.IOException: Not an Avro data file
Can you please let me know why is my configuration not working ?

Related

How to optimize cloning in new roblox update

Everything has been going smooth until the new roblox update and now my barrage effect clones make the game insanely laggy.
Any help on optimizing it would be greatly appreciated.
I know this might sound lazy but it would be great if you could optimize it for me. I have 0 idea how to optimize and this is my last hope.
local EffectModule = require(game.ServerStorage:WaitForChild("HitEffectModule"))
local event = game:GetService("ReplicatedStorage"):WaitForChild("TheWorld").Barrage -- this is the event which fires from client to server
local TweenService = game:GetService("TweenService") -- this is a very useful function for extra effects
local Debris = game:GetService("Debris")
local Damage = 1.8
local Punched = game.Workspace.Sounds.Punched
--u can get ur own one im just using a free one as an example
local function Detect(Character)
local Hitbox = script.Hitbox:Clone()
Hitbox.Parent = workspace
Hitbox.CFrame = Character.PrimaryPart.CFrame * CFrame.new(0,0,Hitbox.Size.Z/2 * -1)
Debris:AddItem(Hitbox,.3)
local Connection
Connection = Hitbox.Touched:Connect(function(hitted)
if hitted.Parent:FindFirstChild("Humanoid") and not hitted:IsDescendantOf(Character) then
local Enemy = hitted.Parent
Enemy.Humanoid:TakeDamage(Damage)
Connection:Disconnect()
Hitbox:Destroy()
local bv = Instance.new("BodyVelocity")
bv.MaxForce = Vector3.new(1e6,1e6,1e6)
bv.Velocity = Character.PrimaryPart.CFrame.LookVector * 10
Debris:AddItem(bv,.2)
bv.Parent = Enemy.PrimaryPart
EffectModule.Gore2(Enemy.UpperTorso)
Enemy.Humanoid:LoadAnimation(script.HitAnim):Play()
Punched:Play()
Enemy.Humanoid.WalkSpeed = 3
wait(1)
Enemy.Humanoid.WalkSpeed = 16
end
end)
end
local function CreateArm(Character)
local Stand = game:GetService("ServerStorage").StandModel.TheWorld
local StartXR = math.random(150,350)/100
Detect(Character)
local StartXL = math.random(150,350)/100 * -1
local StartYR = math.random(-150,250)/100
local StartYL = math.random(-150,250)/100
local LeftArmModel = Instance.new("Model")
LeftArmModel.Name = "LeftArm"
LeftArmModel.Parent = game.Workspace
Debris:AddItem(LeftArmModel,.3)
local LLA = Stand:FindFirstChild("LeftLowerArm"):Clone()
LLA.Parent = LeftArmModel
local LUA = Stand:FindFirstChild("LeftUpperArm"):Clone()
LUA.Parent = LeftArmModel
local LH = Stand:FindFirstChild("LeftHand"):Clone()
LH.Parent = LeftArmModel
local weld1 = Instance.new("WeldConstraint")
weld1.Part0 = LH
weld1.Part1 = LLA
weld1.Parent = LH
local weld2 = Instance.new("WeldConstraint")
weld2.Part0 = LUA
weld2.Part1 = LLA
weld2.Parent = LUA
for i , v in pairs(LeftArmModel:GetDescendants()) do
if v:IsA("Texture") or v:IsA("BasePart") or v:IsA("Decal") then
v.Transparency = .2
TweenService:Create(v,TweenInfo.new(.3),{Transparency = .75}):Play()
end
if v:IsA("Motor6D") then
v:Destroy()
end
end
LeftArmModel.PrimaryPart = LLA
LeftArmModel.PrimaryPart.Anchored = true
LeftArmModel.PrimaryPart.CFrame = CFrame.new(Character.PrimaryPart.CFrame * CFrame.new(StartXL,StartYL,-1.5).p,Character.PrimaryPart.CFrame * CFrame.new(StartXL * .4,StartYL * .4,-8).p) * CFrame.Angles(math.rad(90),0,0)
TweenService:Create(LeftArmModel.PrimaryPart,TweenInfo.new(.3),{CFrame = LeftArmModel.PrimaryPart.CFrame * CFrame.new(0,-5,0)}):Play()
local RightArmModel = Instance.new("Model")
RightArmModel.Name = "LeftArm"
RightArmModel.Parent = game.Workspace
Debris:AddItem(RightArmModel,.3)
local RLA = Stand:FindFirstChild("RightLowerArm"):Clone()
RLA.Parent = RightArmModel
local RUA = Stand:FindFirstChild("RightUpperArm"):Clone()
RUA.Parent = RightArmModel
local RH = Stand:FindFirstChild("RightHand"):Clone()
RH.Parent = RightArmModel
local weld1 = Instance.new("WeldConstraint")
weld1.Part0 = RH
weld1.Part1 = RLA
weld1.Parent = RH
local weld2 = Instance.new("WeldConstraint")
weld2.Part0 = RUA
weld2.Part1 = RLA
weld2.Parent = RUA
for i , v in pairs(RightArmModel:GetDescendants()) do
if v:IsA("Texture") or v:IsA("BasePart") or v:IsA("Decal") then
v.Transparency = .2
TweenService:Create(v,TweenInfo.new(.3),{Transparency = .75}):Play()
end
if v:IsA("Motor6D") then
v:Destroy()
end
end
RightArmModel.PrimaryPart = RLA
RightArmModel.PrimaryPart.Anchored = true
RightArmModel.PrimaryPart.CFrame = CFrame.new(Character.PrimaryPart.CFrame * CFrame.new(StartXR,StartYR,-1.5).p,Character.PrimaryPart.CFrame * CFrame.new(StartXR * .4,StartYR * .4,-8).p) * CFrame.Angles(math.rad(90),0,0)
TweenService:Create(RightArmModel.PrimaryPart,TweenInfo.new(.3),{CFrame = RightArmModel.PrimaryPart.CFrame * CFrame.new(0,-5,0)}):Play()
end
event.OnServerEvent:Connect(function(Player,hold)
local Character = Player.Character
local Stand = Character:FindFirstChild(Character.Name)
if Stand then
if not Character:FindFirstChild("Barraging") then
if hold == true and Character.UsingMove.Value == false then
local barraging = Instance.new("BoolValue")
barraging.Name = "Barraging"
barraging.Parent = Character
Character.UsingMove.Value = true
Character.PrimaryPart.StandPosition.Position = Vector3.new(0,-1,-2)
local Anim = Stand.AnimationController:LoadAnimation(script.Anim)
Anim:Play()
local Sound = script.BarrageSound:Clone()
Sound.Parent = Character.PrimaryPart
Sound:Play()
local Starttime = tick()
repeat
wait(.30)
CreateArm(Character)
until tick() - Starttime >= 5 or not Character:FindFirstChild("Barraging")
Character.PrimaryPart.StandPosition.Position = Vector3.new(-2,0.7,3)
Anim:Stop()
Sound:Destroy()
Character.UsingMove.Value = false
end
else
Character:FindFirstChild("Barraging"):Destroy()
end
end
end)
Instead of cloning it from a storage, at the start just clone a bunch of them and keep them very high up in sky so that it wont be rendered, and as you need them position them accordingly while switching them.

RDP requesting password

I'm working on a RDP application, but windows is ignoring my password provided in the code and is showing the prompt for password input, here is my code:
rdp.Server = server
rdp.UserName = user
rdp.TransportSettings3.GatewayHostname = gateway
rdp.TransportSettings3.GatewayUsageMethod = 1
rdp.TransportSettings3.GatewayCredsSource = 4
rdp.TransportSettings3.GatewayProfileUsageMethod = 1
rdp.AdvancedSettings9.AuthenticationLevel = 0
rdp.AdvancedSettings9.EnableAutoReconnect = True
rdp.AdvancedSettings9.DisplayConnectionBar = True
Dim IRCNS As MSTSCLib.IMsRdpClientNonScriptable5 =
DirectCast(rdp.GetOcx(), IMsTscNonScriptable)
IRCNS.PromptForCredentials = False
IRCNS.AllowPromptingForCredentials = False
IRCNS.ClearTextPassword = pwd
IRCNS.PromptForCredsOnClient = False
rdpView.Connect()

vb6 to vb.net for OracleInProcServer.OraDynaset

I am converting a project which is coded in old vb6 way. I am replacing the instances of OracleInProcServer.OraDynaset to Datatable for getting the data from database. But a form is generated in a dynamic way has the below code. i don't know how be there a alternate code for there in vb.net. Is there any way I can replace the below code without using dsAnswer As OracleInProcServer.OraDynaset. I don't know what is ! here also and how it is working.
Dim dsAnswer As OracleInProcServer.OraDynaset
Select Case dsAnswer!Shape.Value
Case 0 To 5
iShapes = iShapes + 1
'Load(f!shpCtrl(iShapes))
frmDynaForm.shpCtrl.Load(iShapes)
frmDynaForm.shpCtrl(iShapes).FillColor = System.Drawing.Color.FromArgb(DecodeColorDesc(Format$(dsAnswer!Color.Value)))
'frmDynaForm.shpCtrl(iShapes).Shape = dsAnswer!Shape.Value
frmDynaForm.shpCtrl(iShapes).Top = dsAnswer!Top.Value
frmDynaForm.shpCtrl(iShapes).Left = dsAnswer!Left.Value
frmDynaForm.shpCtrl(iShapes).Height = dsAnswer!Height.Value
frmDynaForm.shpCtrl(iShapes).Width = dsAnswer!Width.Value
frmDynaForm.shpCtrl(iShapes).BorderWidth = dsAnswer!BorderWidth.Value
frmDynaForm.shpCtrl(iShapes).Visible = True
Case -1
iLines = iLines + 1
'Load(f!lineCtrl(iLines))
frmDynaForm.lineCtrl.Load(iShapes)
frmDynaForm.lineCtrl(iLines).BorderColor = System.Drawing.Color.FromArgb(DecodeColorDesc(Format$(dsAnswer!Color.Value)))
frmDynaForm.lineCtrl(iLines).Y1 = dsAnswer!Top.Value
frmDynaForm.lineCtrl(iLines).X1 = dsAnswer!Left.Value
frmDynaForm.lineCtrl(iLines).Y2 = dsAnswer!Height.Value
frmDynaForm.lineCtrl(iLines).X2 = dsAnswer!Width.Value
frmDynaForm.lineCtrl(iLines).BorderWidth = dsAnswer!BorderWidth.Value
frmDynaForm.lineCtrl(iLines).Visible = True
End Select

Sending Commands to a serialport

I'm trying to send this to SerialPort to command the devices' LED to lit on.
SData(0) = 85
SData(1) = 170
SData(2) = 36
SData(3) = 1
SData(4) = 2
SData(5) = 0
SData(6) = 1
SData(7) = 0
SData(8) = 0
SData(9) = 0
SData(10) = 0
SData(11) = 0
SData(12) = 0
SData(13) = 0
SData(14) = 0
SData(15) = 0
SData(16) = 0
SData(17) = 0
SData(18) = 0
SData(19) = 0
SData(20) = 0
SData(21) = 0
SData(22) = 39
SData(23) = 1
SerialPort1.Open()
For i = 0 To 23
SerialPort1.Write(Chr(SData(i)))
Next
SerialPort1.Close()
I don't get response from the device or anything. Then i noticed that the module freezes for 2 seconds, i think it's the loop.
I think i need to know the steps of how a SerialPort runs from open to close.
Btw, there's a original copy of this but it was in VB6
here's whats working: MSComm1.Output = Chr(SData(i))

Error in assigning object of nullable type in vb.net

I am new to VB.net and facing a strange situation. I have a structure that contains another structure. The inner structure is nullable. Now wht I do is something like the following. I create a new instance of the inner structure, set the member variables and assign the whole structure to the inner structure of the parent. But it is giving an error. The assignment is not successful. If I try to peek into the structure in the watch window, it says "property evaluation failed" for the HasValue and Value properties.
Dim testData As List(Of TestData) = Nothing
Dim testData_List1 As New TestData
With testData_List1.commonTestParam
.AccuchekActiveEnergy = 2.56
.AccuchekActiveEnergyUnit = ActiveEnergyUnit.KiloWattHour
.AccuchekApparentEnergy = 34.56
.AccuchekApparentEnergyUnit = ApparentEnergyUnit.VoltAmpereHour
.AccuchekFrequency = 1
.AccuchekRange = "20474 ewr 34324"
.AccuchekType = AccuchekType.AccuchekOne
.ActiveLoadRPhase = 145
.AvgActiveLoad = 2.56
.AvgActiveLoadUnit = ActiveLoadUnit.Watt
.AvgPowerFactor = 0
.AvgPowerFactorType = PowerFactorType.PFLag
.ConditionalFlag1 = 0
.ConsumerNo = "343122050242"
.CurrentRPhase = 1
.GeneralFlag1 = 0
.InstantaneousPFRPhase = 2
.ManufacturingYear = 2009
.MeterActiveEnergy = 258.89
.MeterActiveEnergyUnit = ActiveEnergyUnit.KiloWattHour
.MeterActiveError = 20
.MeterApparentError = 14
.MeterConstant = 3200
.MeterConstantUnit = MeterConstantUnit.RevsPerkVAh
.MeterMake = "DS"
.MeterSNo = "6563402"
.MeterTypeAndClass = MeterTypeAndClass.MTCElectorMechWith20
.MTFID = "123456789"
.NoofTestRevolutions = 100
.PulseCriteria = 0
.RatedBasic = 25
.RatedMax = 30
.RatedVoltage = 15
.ReactiveCurrentRPhase = 145
.RemarkID = 0
.TestDateAndTime = "100320101545"
.TestDuration = 2145
.TesterCode = 0
.TestID = "147852"
.TestMode = TestMode.TMOpticalScanner
.TestNumber = 0
.VoltageRPhase = 145
End With
Dim accuchek3TestParameters1 As New Accuchek3PhaseTestParameters
With accuchek3TestParameters1
.AccuchekReactiveLagEnergy = 2.46
.AccuchekReactiveLagEnergyUnit = ReactiveEnergyUnit.KiloVoltAmpereReactiveHour
.AccuchekReactiveLeadEnergy = 2.56
.AccuchekReactiveLeadEnergyUnit = ReactiveEnergyUnit.KiloVoltAmpereReactiveHour
.ActiveLoadBPhase = 14
.ActiveLoadYPhase = 15
.AvgApparentLoad = 10
.AvgApparentLoadUnit = ApparentLoadUnit.KiloVoltAmpere
.AvgReactiveLagLoad = 14
.AvgReactiveLagLoadUnit = ReactiveLoadUnit.KiloVoltAmpereReactive
.AvgReactiveLeadLoad = 15
.AvgReactiveLeadLoadUnit = ReactiveLoadUnit.KiloVoltAmpereReactive
.ConditionalFlag2 = 0
.ConditionalFlag3 = 0
.CTRatio = 1.23
.CurrentBPhase = 10
.CurrentYPhase = 11
.InstantaneousPFBPhase = 0
.InstantaneousPFYPhase = 1
.MeterApparentEnergy = 1.01
.MeterApparentUnit = ApparentEnergyUnit.KiloVoltAmpereHour
.MeterReactiveLagEnergy = 1.25
.MeterReactiveLagError = 1.25
.MeterReactiveLagUnit = ReactiveEnergyUnit.KiloVoltAmpereReactiveHour
.MeterReactiveLeadEnergy = 1.45
.MeterReactiveLeadError = 1.56
.MeterReactiveLeadUnit = ReactiveEnergyUnit.KiloVoltAmpereReactiveHour
.PercentageLoad = 1
.PTRatio = 1
.ReactiveCurrentBPhase = 10
.ReactiveCurrentYPhase = 11
.VoltageBPhase = 10
.VoltageYPhase = 10
End With
testData_List1.accuchek3TestParameters = accuchek3TestParameters1
testData.Add(testData_List1)
Can somebody please guide me?
Your first line is:
Dim testData As List(Of TestData) = Nothing
Then at the bottom you do
testData.Add(testData_List1)
I can't see anywhere in between where you do something like:
testData = New List(Of TestData)()
Though it's slightly hard to read since you've got both a variables and types with TestData as their name (or part of their name) so I might just be missing that.