javascript - nightmare js browsing and parsing multiple sites - Stack Overflow

admin管理员组

文章数量:1026989

I'm trying to automate 2 sites with github/segmentio/nightmare

Example:

1. I want to go on site_1 (temporary email service) and get an email.
2. Then go to site_2 and fill that email, then submit a form.
3. Then go to to site_1 and check if email arrived
4. Parse the email message for activation link
5. Then click that link and go to site_2 again.

........... etc

Is Nightmare js and node.js good for this kind of browser manipulation?

Because it's ok for simple things like on official API:

var Nightmare = require('nightmare');
new Nightmare()
.goto('')
.type('input[title="Search"]', 'github nightmare')
.click('.searchsubmit')
.run(function (err, nightmare) {
  if (err) return console.log(err);
  console.log('Done!');
});

What about plex things? Thank you

I'm trying to automate 2 sites with github./segmentio/nightmare

Example:

1. I want to go on site_1 (temporary email service) and get an email.
2. Then go to site_2 and fill that email, then submit a form.
3. Then go to to site_1 and check if email arrived
4. Parse the email message for activation link
5. Then click that link and go to site_2 again.

........... etc

Is Nightmare js and node.js good for this kind of browser manipulation?

Because it's ok for simple things like on official API:

var Nightmare = require('nightmare');
new Nightmare()
.goto('http://yahoo.')
.type('input[title="Search"]', 'github nightmare')
.click('.searchsubmit')
.run(function (err, nightmare) {
  if (err) return console.log(err);
  console.log('Done!');
});

What about plex things? Thank you

• javascript
• node.js
• nightmare

Share Improve this question Follow edited Sep 11, 2015 at 8:33 DoctorP asked Sep 11, 2015 at 8:17 DoctorPDoctorP 16922 silver badges1010 bronze badges 4

• So you just copy paste the first example? Just under this, it has all it's API. – GramThanos Commented Sep 11, 2015 at 8:26
• It's from official API. Saying that simple things are ok. But what if I want to browse 2, 4 pages and evaluate (copy paste) data from one to another? – DoctorP Commented Sep 11, 2015 at 8:31
• This is a plugin for the phantomjs . Did you searched it there? – GramThanos Commented Sep 11, 2015 at 8:35
• Yes I'm using it but it harder to manipulate DOM with phantomjs: fill forms. click. select etc. Nightmare is easier. – DoctorP Commented Sep 11, 2015 at 8:39

Add a ment  | 

2 Answers 2

Sorted by: Reset to default 3

Here I create a function to get html sources for different conditions, I'm crawling the TimeWarnerCable page to get info about TV, Internet and Bundle plans, so my function get some parameters and react for each one in different calls:

function getSource(url,serviceQuantity,zip){
  var defer=Q.defer();
  var Nightmare = require('nightmare');
  var nightmare = Nightmare({openDevTools:browserDev ,show: browserVisible,'webPreferences':{partition: 'nopersist'}});

  nightmare
  .goto(url)
  .cookies.clear()
  .wait(2000)
  .exists('div.messagebox-wrapper.twc-container[style="display: block;"]')
  .then(function(noZipSet){
    if (noZipSet){
      debug('No zipcode settled down');
      nightmare
      .insert('fieldset > div > input[placeholder="Enter Your ZIP Code"]',zip)
      .type('fieldset > div > input[placeholder="Enter Your ZIP Code"]', '\u000d');//I do "Enter" because nightmare can't find the submit button
    }else{
      debug('Zipcode settled down');
      nightmare
      .click('div.section.newHeaderIcons > div > ul > li:nth-child(4) > div > a')
      .wait(2000)
      .insert('form.geoLoc > fieldset > div > input[placeholder="Update Your ZIP Code"]',zip)
      .type('form.geoLoc > fieldset > div > input[placeholder="Update Your ZIP Code"]', '\u000d');//I do "Enter" because nightmare can't find the submit button
    }
    nightmare
    .wait(8500)
    .exists('div[style="display: block;"] > div > div > div > div > div > div > div.parsys.oof-error-content > div > div > div > div > div > div > p[style="color: #333333;"]')
    .then(function(zipNotAvailable){
      if (zipNotAvailable){
        debug('Service not available in '+zip+' for '+serviceQuantity+' services');
        nightmare
          .end()
          .then(function(){
            defer.resolve('');
          });
      }else{
        debug('Service available on the zipcode');
      switch (serviceQuantity) {
          case 1:
              nightmare
                  .evaluate(function(){
                    return document.querySelector('html').innerHTML;
                  })
                  .end()
                  .then(function (result) {
                    defer.resolve(result);
                  })
                  .catch(function (error) {
                    debug('ERROR >> Search failed:', error);
                  });
              break;
          case 2:
              nightmare
                .click('#tv-filter')
                .wait(500)
                .click('#internet-filter')
                .wait(500)
                .evaluate(function(){
                  return document.querySelector('html').innerHTML;
                })
                .end()
                .then(function (result) {
                   defer.resolve(result);
                })
                .catch(function (error) {
                   debug('ERROR >> Search failed:', error);
                });
              break;
          case 3:
              nightmare
                  .click('#tv-filter')
                  .wait(500)
                  .click('#internet-filter')
                  .wait(500)
                  .click('#phone-filter')
                  .wait(500)
                  .evaluate(function(){
                    return document.querySelector('html').innerHTML;
                  })
                  .end()
                  .then(function (result) {
                    defer.resolve(result);
                  })
                  .catch(function (error) {
                    debug('ERROR >> Search failed:', error);
                  });
                  break;
       }
      }
    });
  });
  return defer.promise;
}

Nightmare now uses Electron instead of PhantomJS and the API is different to what it used to be. Documentation only shows how to use with generators. Here's how I would do what you are wanting without the use of generators, and using promises instead. You can kill the electron browser using these three mands when you're done.

Also, make sure you validate each step of DOM manipulation (with controlled recursion) and leave plenty of wait times between operations. Otherwise you won't always get what you're expecting.

var Nightmare   = require('nightmare'),
    Promise     = require('q').Promise;

var nightmare = new Nightmare({show: true});

function goToSite1(data){
    Promise.resolve(nightmare
        .goto('http://stackoverflow./questions/32518777/nightmare-js-browsing-and-parsing-multiple-sites')
        .evaluate(function(){
            return $('#question-header > h1 > a').text();
        })).then(function(html){
            goToSite2(html);
        });
}

function goToSite2(data){
    Promise.resolve(nightmare
        .goto('https://github./segmentio/nightmare')
    ).then(function(){
        nightmare.proc.disconnect();
        nightmare.proc.kill();
        nightmare.ended = true;
        doOtherStuff(data);
    })
}

function doOtherStuff(data){
    console.log(data)
}

goToSite1();

I'm trying to automate 2 sites with github/segmentio/nightmare

Example:

1. I want to go on site_1 (temporary email service) and get an email.
2. Then go to site_2 and fill that email, then submit a form.
3. Then go to to site_1 and check if email arrived
4. Parse the email message for activation link
5. Then click that link and go to site_2 again.

........... etc

Is Nightmare js and node.js good for this kind of browser manipulation?

Because it's ok for simple things like on official API:

var Nightmare = require('nightmare');
new Nightmare()
.goto('')
.type('input[title="Search"]', 'github nightmare')
.click('.searchsubmit')
.run(function (err, nightmare) {
  if (err) return console.log(err);
  console.log('Done!');
});

What about plex things? Thank you

I'm trying to automate 2 sites with github./segmentio/nightmare

Example:

1. I want to go on site_1 (temporary email service) and get an email.
2. Then go to site_2 and fill that email, then submit a form.
3. Then go to to site_1 and check if email arrived
4. Parse the email message for activation link
5. Then click that link and go to site_2 again.

........... etc

Is Nightmare js and node.js good for this kind of browser manipulation?

Because it's ok for simple things like on official API:

var Nightmare = require('nightmare');
new Nightmare()
.goto('http://yahoo.')
.type('input[title="Search"]', 'github nightmare')
.click('.searchsubmit')
.run(function (err, nightmare) {
  if (err) return console.log(err);
  console.log('Done!');
});

What about plex things? Thank you

• javascript
• node.js
• nightmare

Share Improve this question Follow edited Sep 11, 2015 at 8:33 DoctorP asked Sep 11, 2015 at 8:17 DoctorPDoctorP 16922 silver badges1010 bronze badges 4

• So you just copy paste the first example? Just under this, it has all it's API. – GramThanos Commented Sep 11, 2015 at 8:26
• It's from official API. Saying that simple things are ok. But what if I want to browse 2, 4 pages and evaluate (copy paste) data from one to another? – DoctorP Commented Sep 11, 2015 at 8:31
• This is a plugin for the phantomjs . Did you searched it there? – GramThanos Commented Sep 11, 2015 at 8:35
• Yes I'm using it but it harder to manipulate DOM with phantomjs: fill forms. click. select etc. Nightmare is easier. – DoctorP Commented Sep 11, 2015 at 8:39

Add a ment  | 

2 Answers 2

Sorted by: Reset to default 3

Here I create a function to get html sources for different conditions, I'm crawling the TimeWarnerCable page to get info about TV, Internet and Bundle plans, so my function get some parameters and react for each one in different calls:

function getSource(url,serviceQuantity,zip){
  var defer=Q.defer();
  var Nightmare = require('nightmare');
  var nightmare = Nightmare({openDevTools:browserDev ,show: browserVisible,'webPreferences':{partition: 'nopersist'}});

  nightmare
  .goto(url)
  .cookies.clear()
  .wait(2000)
  .exists('div.messagebox-wrapper.twc-container[style="display: block;"]')
  .then(function(noZipSet){
    if (noZipSet){
      debug('No zipcode settled down');
      nightmare
      .insert('fieldset > div > input[placeholder="Enter Your ZIP Code"]',zip)
      .type('fieldset > div > input[placeholder="Enter Your ZIP Code"]', '\u000d');//I do "Enter" because nightmare can't find the submit button
    }else{
      debug('Zipcode settled down');
      nightmare
      .click('div.section.newHeaderIcons > div > ul > li:nth-child(4) > div > a')
      .wait(2000)
      .insert('form.geoLoc > fieldset > div > input[placeholder="Update Your ZIP Code"]',zip)
      .type('form.geoLoc > fieldset > div > input[placeholder="Update Your ZIP Code"]', '\u000d');//I do "Enter" because nightmare can't find the submit button
    }
    nightmare
    .wait(8500)
    .exists('div[style="display: block;"] > div > div > div > div > div > div > div.parsys.oof-error-content > div > div > div > div > div > div > p[style="color: #333333;"]')
    .then(function(zipNotAvailable){
      if (zipNotAvailable){
        debug('Service not available in '+zip+' for '+serviceQuantity+' services');
        nightmare
          .end()
          .then(function(){
            defer.resolve('');
          });
      }else{
        debug('Service available on the zipcode');
      switch (serviceQuantity) {
          case 1:
              nightmare
                  .evaluate(function(){
                    return document.querySelector('html').innerHTML;
                  })
                  .end()
                  .then(function (result) {
                    defer.resolve(result);
                  })
                  .catch(function (error) {
                    debug('ERROR >> Search failed:', error);
                  });
              break;
          case 2:
              nightmare
                .click('#tv-filter')
                .wait(500)
                .click('#internet-filter')
                .wait(500)
                .evaluate(function(){
                  return document.querySelector('html').innerHTML;
                })
                .end()
                .then(function (result) {
                   defer.resolve(result);
                })
                .catch(function (error) {
                   debug('ERROR >> Search failed:', error);
                });
              break;
          case 3:
              nightmare
                  .click('#tv-filter')
                  .wait(500)
                  .click('#internet-filter')
                  .wait(500)
                  .click('#phone-filter')
                  .wait(500)
                  .evaluate(function(){
                    return document.querySelector('html').innerHTML;
                  })
                  .end()
                  .then(function (result) {
                    defer.resolve(result);
                  })
                  .catch(function (error) {
                    debug('ERROR >> Search failed:', error);
                  });
                  break;
       }
      }
    });
  });
  return defer.promise;
}

Nightmare now uses Electron instead of PhantomJS and the API is different to what it used to be. Documentation only shows how to use with generators. Here's how I would do what you are wanting without the use of generators, and using promises instead. You can kill the electron browser using these three mands when you're done.

Also, make sure you validate each step of DOM manipulation (with controlled recursion) and leave plenty of wait times between operations. Otherwise you won't always get what you're expecting.

var Nightmare   = require('nightmare'),
    Promise     = require('q').Promise;

var nightmare = new Nightmare({show: true});

function goToSite1(data){
    Promise.resolve(nightmare
        .goto('http://stackoverflow./questions/32518777/nightmare-js-browsing-and-parsing-multiple-sites')
        .evaluate(function(){
            return $('#question-header > h1 > a').text();
        })).then(function(html){
            goToSite2(html);
        });
}

function goToSite2(data){
    Promise.resolve(nightmare
        .goto('https://github./segmentio/nightmare')
    ).then(function(){
        nightmare.proc.disconnect();
        nightmare.proc.kill();
        nightmare.ended = true;
        doOtherStuff(data);
    })
}

function doOtherStuff(data){
    console.log(data)
}

goToSite1();

本文标签： javascriptnightmare js browsing and parsing multiple sitesStack Overflow