SatChanGen: KingOfSat.cs@29ccfbf98e54

     1 using System;

     2 using System.Collections.Generic;

     3 using System.Linq;

     4 using System.Text;

     5 using System.Threading.Tasks;

     6 using CsQuery;

     7 using System.Diagnostics;

     8 using System.Net;

    10 namespace SatChanGen

    11 {

    12     class KingOfSat

    13     {

    14         //

    15         // Summary:

    16         //     Create a new CQ object wrapping a single element.

    17         //

    18         // Parameters:

    19         //   aUrl:

    20         //     URL to a KingOfSat channel list. Typically a package list.

    21         //

    22         // Return:

    23         //   List of channels parsed.

    24         public static List<Channel> Parse(List<Channel> aChannels, string aUrl, string aOrbitalPosition)

    25         {

    26             //To avoid duplicated name

    27             Dictionary<string, int> names = new Dictionary<string, int>();

    29             string kos = new WebClient().DownloadString(aUrl);

    30             //Debug.Write(kos);

    32             CQ dom = kos;

    34             //Get all the Frequency elements in our page

    35             CQ sats = dom[".frq"];

    37             //Create our list of channels

    38             List<Channel> channels = new List<Channel>();

    40             foreach (IDomObject frq in sats.ToList())

    41             {

    42                 Channel common = new Channel();

    44                 //Parse channel details

    45                 common.OrbitalPosition = WebUtility.HtmlDecode(frq.Cq().Find("tbody > tr > td > a > font").Get(0).InnerText).Trim();

    46                 if (common.OrbitalPosition != aOrbitalPosition)

    47                 {

    48                     //Wrong sat, skip

    49                     continue;

    50                 }

    51                 common.Satellite = WebUtility.HtmlDecode(frq.Cq().Find("tbody > tr > td:nth-child(2) > a").Get(0).InnerText);

    52                 common.Frequency = WebUtility.HtmlDecode(frq.Cq().Find("tbody > tr > td:nth-child(3)").Get(0).InnerText);

    53                 common.Polarisation = WebUtility.HtmlDecode(frq.Cq().Find("tbody > tr > td:nth-child(4)").Get(0).InnerText);

    54                 common.Transponder = WebUtility.HtmlDecode(frq.Cq().Find("tbody > tr > td:nth-child(5) > a").Get(0).InnerText);

    55                 common.Beam = WebUtility.HtmlDecode(frq.Cq().Find("tbody > tr > td:nth-child(6) > a").Get(0).InnerText);

    56                 common.Standard = WebUtility.HtmlDecode(frq.Cq().Find("tbody > tr > td:nth-child(7)").Get(0).InnerText);

    57                 common.Modulation = WebUtility.HtmlDecode(frq.Cq().Find("tbody > tr > td:nth-child(8)").Get(0).InnerText);

    58                 common.SymbolRate = WebUtility.HtmlDecode(frq.Cq().Find("tbody > tr > td:nth-child(9) > a").Get(0).InnerText);

    59                 common.FEC = WebUtility.HtmlDecode(frq.Cq().Find("tbody > tr > td:nth-child(9) > a:nth-child(2)").Get(0).InnerText);

    60                 try

    61                 {

    62                     common.Provider = WebUtility.HtmlDecode(frq.Cq().Find("tbody > tr > td:nth-child(10) > b").Get(0).InnerText);

    63                 }

    64                 catch (Exception)

    65                 {

    66                 }

    68                 common.Bitrate = WebUtility.HtmlDecode(frq.Cq().Find("tbody > tr > td:nth-child(10)").Get(0).InnerText);

    69                 if (common.Bitrate.Substring(0, ", ".Length) == ", ")

    70                 {

    71                     common.Bitrate = common.Bitrate.Substring(", ".Length, common.Bitrate.Length - ", ".Length);

    72                 }

    73                 //

    74                 common.NetworkID = WebUtility.HtmlDecode(frq.Cq().Find("tbody > tr > td:nth-child(11)").Get(0).InnerText);

    75                 common.NetworkID = common.NetworkID.Substring("NID:".Length, common.NetworkID.Length - "NID:".Length);

    76                 //

    77                 common.TransponderID = WebUtility.HtmlDecode(frq.Cq().Find("tbody > tr > td:nth-child(12)").Get(0).InnerText);

    78                 common.TransponderID = common.TransponderID.Substring("TID:".Length, common.TransponderID.Length - "TID:".Length);

    80                 //We got common properties for the coming channels

    81                 //Debug.Write(common.ToString());

    83                 //Now get all the channels for that frequency

    84                 //Channel common = new Channel();

    86                 CQ channelsDiv = frq.Cq().Next("div");

    87                 CQ channelsTableRows = channelsDiv.Find("table.fl > tbody").Children("tr");

    89                 foreach (IDomObject row in channelsTableRows)

    90                 {

    91                     Channel channel = new Channel();

    92                     //Initialize this channel with common properties on this frequency

    93                     channel.Copy(common);

    95                     //Try and parse channel name

    96                     CQ cqChannelName = row.Cq().Find("td:nth-child(3) > a");

    97                     if (cqChannelName.Length == 0)

    98                     {

    99                         cqChannelName = row.Cq().Find("td:nth-child(3) > i");

   100                         if (cqChannelName.Length == 0)

   101                         {

   102                             //Can't get channel name

   103                             Debug.Write("WARNING: Can't find channel name! Skipping this channel");

   104                             continue;

   105                         }

   106                     }

   108                     string channelNameInnerText = cqChannelName.Get(0).InnerText;

   109                     channel.Name = WebUtility.HtmlDecode(channelNameInnerText).Trim();

   110                     //Encoding decoder = Encoding.UTF8;

   111                     //channel.Name = decoder.GetString(channel.Name);

   112                     //channel.Name = channelNameInnerText.Trim(); //Make up your mind :)

   113                     if (channel.Name == "Name")

   114                     {

   115                         //Skipping header rows

   116                         continue;

   117                     }

   119                     //Make sure our channel name looks descent

   120                     channel.Name = CleanChannelName(channel.Name);

   121                     //Make sure the resulting name is unique to avoid having multiple tuning detail for a single channel

   122                     if (names.ContainsKey(channel.Name))

   123                     {

   124                         names[channel.Name]++;

   125                         channel.Name += " " + names[channel.Name];

   126                     }

   127                     else

   128                     {

   129                         names.Add(channel.Name, 1);

   130                     }

   132                     //

   133                     //We don't want channels we already have

   134                     Channel existingChannel = aChannels.Find(c => c.Name == channel.Name);

   135                     if (existingChannel!=null)

   136                     {

   137                         continue;

   138                     }

   141                     //So we have a channel name get the other properties then

   142                     channel.Country = WebUtility.HtmlDecode(row.Cq().Find("td:nth-child(4)").Get(0).InnerText).Trim();

   143                     channel.Category = WebUtility.HtmlDecode(row.Cq().Find("td:nth-child(5)").Get(0).InnerText).Trim();

   144                     if (channel.Category=="")

   145                     {

   146                         channel.Category = "Other";

   147                     }

   148                     //Skip the packages

   149                     //Skip the encryptions

   150                     channel.SID = WebUtility.HtmlDecode(row.Cq().Find("td:nth-child(8)").Get(0).InnerText).Trim();

   151                     channel.VPID = WebUtility.HtmlDecode(row.Cq().Find("td:nth-child(9)").Get(0).InnerText).Trim();

   152                     //Skip audios

   153                     channel.PMT = WebUtility.HtmlDecode(row.Cq().Find("td:nth-child(11)").Get(0).InnerText).Trim();

   154                     channel.PCR = WebUtility.HtmlDecode(row.Cq().Find("td:nth-child(11)").Get(0).InnerText).Trim();

   155                     channel.TXT = WebUtility.HtmlDecode(row.Cq().Find("td:nth-child(11)").Get(0).InnerText).Trim();

   157                     //Append that new channel to our list

   158                     channels.Add(channel);

   160                     //Show it in debug output

   161                     Debug.Write(channel);

   162                 } //For each channel

   163             } //For each frequency

   165             return channels;

   166         }

   168         //

   169         public static string CleanChannelName(string aName)

   170         {

   171             aName = aName.Trim();

   172             string[] remove = { " Germany", " Deutschland", " (Germany)", " (Deutschland)" };

   174             foreach (string item in remove)

   175             {

   176                 //if (aName.EndsWith(item))

   177                 if (aName.Contains(item))

   178                 {

   179                     aName = aName.Substring(0, aName.LastIndexOf(item));

   180                     break; //only allow one match at most

   181                 }

   182             }

   183             aName = aName.Trim();

   184             return aName;

   185         }

   187         //

   188         public static List<Channel> CleanChannelList(List<Channel> aChannels)

   189         {

   190             //Create our list of channels

   191             List<Channel> channels = new List<Channel>();

   193             foreach (Channel channel in aChannels)

   194             {

   195                 Channel hdChannel = aChannels.Find(c => c.Name == channel.Name + " HD");

   196                 if (hdChannel==null

   197                     && !(channel.Name.Contains("Bundesliga") && !channel.Name.Contains("HD")) //We don't want non HD bundesliga

   198                     && !(channel.Name.StartsWith("Sky Sport") && !channel.Name.Contains("HD")) //We don't want non HD Sky Sport

   199                     )

   200                 {

   201                     //Patch some missing or bad categories

   202                     if (channel.Name.Contains("Bundesliga")

   203                         || channel.Name.Contains("Sport"))

   204                     {

   205                         channel.Category = "Sport";

   206                     }

   208                     if (channel.Name.Contains("Sky Select"))

   209                     {

   210                         channel.Category = "Pay per view";

   211                     }

   214                     if (channel.Name.StartsWith("Sky Atlantic")

   215                         || channel.Name.StartsWith("SyFy")

   216                         || channel.Name.StartsWith("Fox"))

   217                     {

   218                         channel.Category = "Series";

   219                     }

   221                     if (channel.Name.StartsWith("Sky 3D"))

   222                     {

   223                         channel.Category = "Movies";

   224                     }

   226                     //Collapse some categories

   227                     if (channel.Category == "Entertainment"

   228                         || channel.Category == "Music"

   229                         || channel.Name.Contains("Music"))

   230                     {

   231                         channel.Category = "General";

   232                     }

   234                     if (channel.Category == "Porn")

   235                     {

   236                         channel.Category = "Erotic";

   237                     }

   239                     if (channel.Category == "Presentations")

   240                     {

   241                         channel.Category = "News";

   242                     }

   244                     if (channel.Category == "History")

   245                     {

   246                         channel.Category = "Documentaries";

   247                     }

   249                     if (channel.Category == "Lifestyle")

   250                     {

   251                         channel.Category = "General";

   252                     }

   254                     //if (channel.Category == "Regional")

   255                     //{

   256                     //    channel.Category = "General";

   257                     //}

   259                     if (channel.Category == "Other")

   260                     {

   261                         channel.Category = "General";

   262                     }

   264                     if (channel.Category == "Cultural")

   265                     {

   266                         channel.Category = "General";

   267                     }

   270                     //No corresponding HD channel, keep it then

   271                     channels.Add(channel);

   272                 }

   273                 else

   274                 {

   275                     Debug.Write("WARNING: Found HD channel for " + channel.Name + ". Discarding it!\n");

   276                 }

   277             }

   279             return channels;

   280         }

   281     }

   282 }

author	StephaneLenclud
	Sat, 16 May 2015 01:31:32 +0200
changeset 5	29ccfbf98e54
parent 4	8372aa8d6292
child 6	3061de2306d9
permissions	-rw-r--r--