I am trying to get the details of my subscribed courses with the link subscribed courses (/api-2.0/users/me/subscribed-courses) using the Udemy API despite this call is not specified on the details, but when calling this on the browser it works great, however when calling the information from code I get error 403 Forbidden.
When calling for the information of a course, this works on code, and on browser.
Apparently i believe it is because of the clouflare protection.
I have the following code, where I was trying to get the accessToken to do the call after making a successful login, but i still get error 403.
{
"udemy": {
"Username": "[email protected]",
"Password": "user_password",
"Client Id": "user_client_id",
"Client Secret": "user_client_secret",
"LoginPage": "www.udemy.com/join/login-popup/",
"APICoursesSubscribed": "www.udemy.com/api-2.0/users/me/subscribed-courses/",
"APICourseDetail": "www.udemy.com/api-2.0/courses/{courseId}"
}
}
using CloudProxySharp;
using OpenQA.Selenium.Chrome;
using OpenQA.Selenium;
using PuppeteerSharp;
using Udemy.API;
using OpenQA.Selenium.Support.UI;
using System.Collections.ObjectModel;
class Program
{
private static APICalls aPICalls = new APICalls();
static async Task Main(string[] args)
{
//Get course details by ID
int courseId = 120042;
var courseDetails = await aPICalls.GetCourseAsync(courseId);
Console.WriteLine(courseDetails.ToString());
var subscribedCourses = await aPICalls.GetSubscribedCoursesAsync();
Console.WriteLine(subscribedCourses.ToString());
Puppeteer();
await Selenium();
}
private static async Task Selenium()
{
var chromeOptions = new ChromeOptions();chromeOptions.AddArgument(
"user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36");
//chromeOptions.AddArgument("--remote-debugging-port=9222");
chromeOptions.AddExcludedArgument("enable-automation");
chromeOptions.AddAdditionalOption("useAutomationExtension", false);
chromeOptions.AddArgument("--disable-blink-features=AutomationControlled");
IWebDriver driver = new ChromeDriver(chromeOptions);
IJavaScriptExecutor js = (IJavaScriptExecutor)driver;
js.ExecuteScript("Object.defineProperty(navigator, 'webdriver', { get: () => false });");
js.ExecuteScript("Object.defineProperty(navigator, 'languages', { get: () => ['en-US', 'en'] });");
js.ExecuteScript("Object.defineProperty(navigator, 'plugins', { get: () => [1, 2, 3, 4, 5] });");
driver.Navigate().GoToUrl(aPICalls.loginPage);
Random rnd = new Random();
int delay = rnd.Next(2000, 5000); // Random delay between 2-5 seconds
Thread.Sleep(delay);
ReadOnlyCollection<string> windowHandles = driver.WindowHandles;
driver.SwitchTo().Window(windowHandles[0]);
WebDriverWait wait = new WebDriverWait(driver, TimeSpan.FromSeconds(60));
string element = string.Empty;
try
{
element = "Cookies";
IWebElement cookies = wait.Until(drv => drv.FindElement(By.Id("onetrust-accept-btn-handler")));
cookies.Click();
element = "Email";
IWebElement email = wait.Until(drv => drv.FindElement(By.Name("email")));
email.SendKeys(aPICalls.username);
element = "Password";
IWebElement password = wait.Until(drv => drv.FindElement(By.Name("password")));
password.SendKeys(aPICalls.password);
element = "Submit";
IWebElement submitButton = wait.Until(drv => drv.FindElement(By.XPath("//button[@type='submit' and contains(@class, 'helpers--auth-submit-button--W3Tqk') and span[text()='Log in']]")));
submitButton.Click();
}
catch (Exception ex)
{
Console.WriteLine($"Element {element} not found.n" + ex.Message);
}
driver.Quit();
}
private static async void Puppeteer()
{
var ss = await GetAccessTokenVia_Puppeteer();
//not working for udemy
var content = CloudProxySharpCFBypass("www.udemy.com").Result;
CheckingHeadLessChrome().Wait();
}
private static async Task<string> GetAccessTokenVia_Puppeteer()
{
try
{
var options = new LaunchOptions
{
Headless = true,
ExecutablePath = "C:\Program Files\Google\Chrome\Application\chrome.exe"
};
using (var browser = await PuppeteerSharp.Puppeteer.LaunchAsync(options))
using (var page = await browser.NewPageAsync())
{
await page.GoToAsync(aPICalls.loginPage);
// Wait for Cloudflare challenge to complete
await Task.Delay(10000);
await page.WaitForSelectorAsync("input[name=email]", new WaitForSelectorOptions { Visible = true });
await page.WaitForSelectorAsync("input[name=password]", new WaitForSelectorOptions { Visible = true });
await page.TypeAsync("input[name=email]", aPICalls.username);
await page.TypeAsync("input[name=password]", aPICalls.password);
// Submit
await page.ClickAsync("button[type=submit]");
await Task.Delay(5000);
var cookies = await page.GetCookiesAsync();
foreach (var cookie in cookies)
{
if (cookie.Name == "access_token")
{
return cookie.Value;
}
}
throw new Exception("Access token not found.");
}
} catch (Exception ex)
{
Console.WriteLine(ex.ToString());
throw new Exception("Access token not found.");
}
}
public static Task<string> CloudProxySharpCFBypass(string url)
{
var handler = new ClearanceHandler("http://localhost:8191/")
{
//Make sure that the string literal is in single line else it won't work
UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36",
MaxTimeout = 60000
};
var client = new HttpClient(handler);
var content = client.GetStringAsync(url);
return content;
}
// Just to see PuppeteerSharp in action in an headless chrome settings
public static async Task CheckingHeadLessChrome()
{
string outputFile = "Shot.png";
var browserFetcher = new BrowserFetcher();
await browserFetcher.DownloadAsync();
var options = new LaunchOptions()
{
ExecutablePath = @"C:Program FilesGoogleChromeApplicationchrome.exe",
Headless = false,
SlowMo = 10
};
await using var browser = await PuppeteerSharp.Puppeteer.LaunchAsync(options);
await using var page = await browser.NewPageAsync();
await page.GoToAsync("www.udemy.com");
var allContent = await page.GetContentAsync();
var cookies = page.GetCookiesAsync();
await page.GoToAsync("www.udemy.com");
}
}
using Newtonsoft.Json.Linq;
using System.Net;
using System.Net.Http.Headers;
using System.Text;
using System.Text.RegularExpressions;
namespace Udemy.API
{
class APICalls
{
string credentialsFilePath = "res\udemyCredentials.json";
public string clientId;
public string clientSecret;
public string APICoursesSubscribed;
public string APICourseDetail;
public string username;
public string password;
public string loginPage;
public string accessToken;
public APICalls()
{
// Read and parse the JSON file
var credentials = ReadCredentials(credentialsFilePath);
this.clientId = credentials["Client Id"].ToString();
this.clientSecret = credentials["Client Secret"].ToString();
this.APICoursesSubscribed = credentials["APICoursesSubscribed"].ToString();
this.APICourseDetail = credentials["APICourseDetail"].ToString();
this.username = credentials["Username"].ToString();
this.password = credentials["Password"].ToString();
this.loginPage = credentials["LoginPage"].ToString();
// Log in to Udemy and get the access token
this.accessToken = GetAccessTokenViaLogin().Result;
}
static JObject ReadCredentials(string filePath)
{
using (StreamReader reader = new StreamReader(filePath))
{
string json = reader.ReadToEnd();
JObject credentials = JObject.Parse(json);
return (JObject)credentials["udemy"];
}
}
private async Task<string> GetAccessTokenViaLogin()
{
using (HttpClientHandler handler = new HttpClientHandler { UseCookies = true })
using (HttpClient client = new HttpClient(handler))
{
// Step 1: Get the CSRF token from the login page
HttpResponseMessage getLoginPageResponse = await client.GetAsync("https://www.udemy.com/join/login-popup/");
string loginPageContent = await getLoginPageResponse.Content.ReadAsStringAsync();
// Extract the CSRF token from the login page content
var csrfToken = ExtractCsrfToken(loginPageContent);
if (csrfToken == null)
{
Console.WriteLine("CSRF token not found.");
return null;
}
var requestBody = new MultipartFormDataContent
{
{ new StringContent(username), "email" },
{ new StringContent(password), "password" },
{ new StringContent(csrfToken), "csrfmiddlewaretoken" }
};
client.DefaultRequestHeaders.Accept.Clear();
client.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json"));
client.DefaultRequestHeaders.Add("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36");
client.DefaultRequestHeaders.Add("Origin", "https://www.udemy.com");
client.DefaultRequestHeaders.Add("Referer", aPICalls.loginPage+"/?locale=en_US&response_type=html&next=https%3A%2F%2Fwww.udemy.com%2Flogout%2F");
HttpResponseMessage response = await client.PostAsync(aPICalls.loginPage+"/?locale=en_US&next=https%3A%2F%2Fwww.udemy.com%2Flogout%2F&response_type=html&response_type=json", requestBody);
if (response.IsSuccessStatusCode)
{
string content = await response.Content.ReadAsStringAsync();
// Print the response for debugging purposes
Console.WriteLine(content);
// Attempt to extract the access token from cookies if available
foreach (Cookie cookie in handler.CookieContainer.GetCookies(new Uri("https://www.udemy.com")))
{
if (cookie.Name == "access_token")
{
return cookie.Value;
}
}
// If the token is not in the cookies, check the response body
JObject jsonResponse = JObject.Parse(content);
if (jsonResponse.ContainsKey("access_token"))
{
return jsonResponse["access_token"].ToString();
}
}
else
{
Console.WriteLine($"Error obtaining access token: {response.StatusCode}");
}
return null;
}
}
private string ExtractCsrfToken(string htmlContent)
{
// This is a simple example, you might need to adjust it to match the actual HTML structure
var csrfTokenStart = htmlContent.IndexOf("csrfmiddlewaretoken") + 22;
var csrfTokenEnd = htmlContent.IndexOf(""", csrfTokenStart);
if (csrfTokenStart > -1 && csrfTokenEnd > -1)
{
return htmlContent.Substring(csrfTokenStart, csrfTokenEnd - csrfTokenStart);
}
return null;
}
public async Task<JObject> GetSubscribedCoursesAsync()
{
return await GetPage(APICoursesSubscribed);
}
public async Task<JObject> GetCourseAsync(int courseId)
{
string url = APICourseDetail.Replace("{courseId}", courseId.ToString());
return await GetPage(url);
}
public async Task<JObject> GetPage(string url)
{
string encodedClientId = Convert.ToBase64String(Encoding.UTF8.GetBytes(clientId));
string encodedClientSecret = Convert.ToBase64String(Encoding.UTF8.GetBytes(clientSecret));
string authHeader = Convert.ToBase64String(Encoding.UTF8.GetBytes($"{clientId}:{clientSecret}"));
HttpClientHandler handler = new HttpClientHandler();
handler.ServerCertificateCustomValidationCallback = (message, cert, chain, errors) => true;
using (HttpClient customClient = new HttpClient(handler))
{
customClient.DefaultRequestHeaders.Accept.Clear();
customClient.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json"));
customClient.DefaultRequestHeaders.Add("X-Udemy-Client-Id", encodedClientId);
customClient.DefaultRequestHeaders.Add("X-Udemy-Client-Secret", encodedClientSecret);
customClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Basic", authHeader);
HttpResponseMessage response = await customClient.GetAsync(url);
if (response.IsSuccessStatusCode)
{
string result = await response.Content.ReadAsStringAsync();
return JObject.Parse(result);
}
else
{
Console.WriteLine($"Error: {response.StatusCode} ({(int)response.StatusCode})");
return null;
}
}
}
}
}
I tried using Selenium, but upon filling the email and password field i would get an unkown error on udemy login page, and on the console shows severar 403 errors.
I tried using Puppeteer-Sharp, and CloudProxySharp as described here, but no luck either
New contributor
rc vf is a new contributor to this site. Take care in asking for clarification, commenting, and answering.
Check out our Code of Conduct.